diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp index 7bb11b44e..3e83a15ea 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp @@ -18,6 +18,7 @@ #include "lib/mysqlclient/ob_isql_connection_pool.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "common/sql_mode/ob_sql_mode_utils.h" +#include "lib/mysqlclient/ob_dblink_error_trans.h" #ifdef OB_BUILD_DBLINK #include "lib/oracleclient/ob_oci_environment.h" #endif diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index 31a5ba332..bb165b336 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -37,8 +37,11 @@ struct ObSessionDDLInfo final { public: ObSessionDDLInfo() - : ddl_info_(0) - {} + : is_ddl_(false), is_source_table_hidden_(false), is_dest_table_hidden_(false), is_heap_table_ddl_(false), + is_ddl_check_default_value_bit_(false), is_mview_complete_refresh_(false), is_refreshing_mview_(false), + is_retryable_ddl_(false), reserved_bit_(0) + { + } ~ObSessionDDLInfo() = default; void set_is_ddl(const bool is_ddl) { is_ddl_ = is_ddl; } bool is_ddl() const { return is_ddl_; } @@ -78,7 +81,7 @@ public: uint64_t is_mview_complete_refresh_: IS_MVIEW_COMPLETE_REFRESH_BIT; uint64_t is_refreshing_mview_: IS_REFRESHING_MVIEW_BIT; uint64_t is_retryable_ddl_: IS_RETRYABLE_DDL_BIT; - uint64_t reserved_bit : RESERVED_BIT; + uint64_t reserved_bit_ : RESERVED_BIT; }; }; }; diff --git a/src/observer/mysql/ob_query_retry_ctrl.cpp b/src/observer/mysql/ob_query_retry_ctrl.cpp index 345f02f54..d134db188 100644 --- a/src/observer/mysql/ob_query_retry_ctrl.cpp +++ b/src/observer/mysql/ob_query_retry_ctrl.cpp @@ -300,6 +300,10 @@ public: v.retry_type_ = RETRY_TYPE_NONE; } v.no_more_test_ = true; + } else if (v.session_.get_ddl_info().is_retryable_ddl()) { + v.client_ret_ = err; + v.retry_type_ = RETRY_TYPE_NONE; + v.no_more_test_ = true; } else if (is_load_local(v)) { v.client_ret_ = err; v.retry_type_ = RETRY_TYPE_NONE; diff --git a/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp b/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp index ea4848aaa..48f085bfb 100644 --- a/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp +++ b/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp @@ -34,7 +34,7 @@ int ObDirectLoadControlPreBeginExecutor::deserialize() int ObDirectLoadControlPreBeginExecutor::check_args() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_ID == arg_.table_id_ || 0 == arg_.task_id_)) { + if (OB_UNLIKELY(OB_INVALID_ID == arg_.table_id_ || 0 == arg_.ddl_param_.task_id_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg_)); } @@ -65,15 +65,9 @@ int ObDirectLoadControlPreBeginExecutor::process() param.avail_memory_ = arg_.avail_memory_; param.write_session_count_ = arg_.write_session_count_; param.exe_mode_ = arg_.exe_mode_; - ObTableLoadDDLParam ddl_param; - uint64_t data_version = 0; - ddl_param.dest_table_id_ = arg_.dest_table_id_; - ddl_param.task_id_ = arg_.task_id_; - ddl_param.schema_version_ = arg_.schema_version_; - ddl_param.snapshot_version_ = arg_.snapshot_version_; - ddl_param.data_version_ = arg_.data_version_; - ddl_param.cluster_version_ = arg_.cluster_version_; - if (OB_FAIL(create_table_ctx(param, ddl_param, table_ctx))) { + param.method_ = arg_.method_; + param.insert_mode_ = arg_.insert_mode_; + if (OB_FAIL(create_table_ctx(param, arg_.ddl_param_, table_ctx))) { LOG_WARN("fail to create table ctx", KR(ret)); } if (OB_SUCC(ret)) { diff --git a/src/observer/table_load/control/ob_table_load_control_rpc_struct.cpp b/src/observer/table_load/control/ob_table_load_control_rpc_struct.cpp index 5a67692aa..efdad2e66 100644 --- a/src/observer/table_load/control/ob_table_load_control_rpc_struct.cpp +++ b/src/observer/table_load/control/ob_table_load_control_rpc_struct.cpp @@ -60,16 +60,12 @@ ObDirectLoadControlPreBeginArg::ObDirectLoadControlPreBeginArg() dup_action_(ObLoadDupActionType::LOAD_INVALID_MODE), px_mode_(false), online_opt_stat_gather_(false), - dest_table_id_(common::OB_INVALID_ID), - task_id_(0), - schema_version_(0), - snapshot_version_(0), - data_version_(0), session_info_(nullptr), avail_memory_(0), write_session_count_(0), exe_mode_(ObTableLoadExeMode::MAX_TYPE), - cluster_version_(0) + method_(ObDirectLoadMethod::INVALID_METHOD), + insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) { free_session_ctx_.sessid_ = ObSQLSessionInfo::INVALID_SESSID; } @@ -84,7 +80,7 @@ ObDirectLoadControlPreBeginArg::~ObDirectLoadControlPreBeginArg() } } -OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadControlPreBeginArg) +OB_DEF_SERIALIZE(ObDirectLoadControlPreBeginArg) { int ret = OB_SUCCESS; LST_DO_CODE(OB_UNIS_ENCODE, @@ -94,11 +90,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadControlPreBeginArg) dup_action_, px_mode_, online_opt_stat_gather_, - dest_table_id_, - task_id_, - schema_version_, - snapshot_version_, - data_version_, + ddl_param_, partition_id_array_, target_partition_id_array_); if (OB_SUCC(ret)) { @@ -109,18 +101,16 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadControlPreBeginArg) OB_UNIS_ENCODE(*session_info_); } } - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_ENCODE, - avail_memory_, - write_session_count_, - exe_mode_, - cluster_version_); - } - + LST_DO_CODE(OB_UNIS_ENCODE, + avail_memory_, + write_session_count_, + exe_mode_, + method_, + insert_mode_); return ret; } -OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadControlPreBeginArg) +OB_DEF_DESERIALIZE(ObDirectLoadControlPreBeginArg) { int ret = OB_SUCCESS; LST_DO_CODE(OB_UNIS_DECODE, @@ -130,11 +120,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadControlPreBeginArg) dup_action_, px_mode_, online_opt_stat_gather_, - dest_table_id_, - task_id_, - schema_version_, - snapshot_version_, - data_version_, + ddl_param_, partition_id_array_, target_partition_id_array_); if (OB_SUCC(ret)) { @@ -144,18 +130,16 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadControlPreBeginArg) OB_UNIS_DECODE(*session_info_); } } - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_DECODE, - avail_memory_, - write_session_count_, - exe_mode_, - cluster_version_); - } - + LST_DO_CODE(OB_UNIS_DECODE, + avail_memory_, + write_session_count_, + exe_mode_, + method_, + insert_mode_); return ret; } -OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadControlPreBeginArg) +OB_DEF_SERIALIZE_SIZE(ObDirectLoadControlPreBeginArg) { int ret = OB_SUCCESS; int64_t len = 0; @@ -166,11 +150,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadControlPreBeginArg) dup_action_, px_mode_, online_opt_stat_gather_, - dest_table_id_, - task_id_, - schema_version_, - snapshot_version_, - data_version_, + ddl_param_, partition_id_array_, target_partition_id_array_); if (OB_SUCC(ret)) { @@ -181,112 +161,110 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadControlPreBeginArg) OB_UNIS_ADD_LEN(*session_info_); } } - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_ADD_LEN, - avail_memory_, - write_session_count_, - exe_mode_, - cluster_version_); - } - + LST_DO_CODE(OB_UNIS_ADD_LEN, + avail_memory_, + write_session_count_, + exe_mode_, + method_, + insert_mode_); return len; } // confirm_begin -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlConfirmBeginArg, - table_id_, - task_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlConfirmBeginArg, + table_id_, + task_id_); // pre_merge -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlPreMergeArg, - table_id_, - task_id_, - committed_trans_id_array_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlPreMergeArg, + table_id_, + task_id_, + committed_trans_id_array_); // start_merge -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlStartMergeArg, - table_id_, - task_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlStartMergeArg, + table_id_, + task_id_); // commit -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlCommitArg, - table_id_, - task_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlCommitArg, + table_id_, + task_id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlCommitRes, - result_info_, - sql_statistics_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlCommitRes, + result_info_, + sql_statistics_); // abort -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlAbortArg, - table_id_, - task_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlAbortArg, + table_id_, + task_id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlAbortRes, - is_stopped_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlAbortRes, + is_stopped_); // get_status -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlGetStatusArg, - table_id_, - task_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlGetStatusArg, + table_id_, + task_id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlGetStatusRes, - status_, - error_code_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlGetStatusRes, + status_, + error_code_); // heartbeat -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlHeartBeatArg, - table_id_, - task_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlHeartBeatArg, + table_id_, + task_id_); // pre_start_trans -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlPreStartTransArg, - table_id_, - task_id_, - trans_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlPreStartTransArg, + table_id_, + task_id_, + trans_id_); // confirm_start_trans -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlConfirmStartTransArg, - table_id_, - task_id_, - trans_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlConfirmStartTransArg, + table_id_, + task_id_, + trans_id_); // pre_finish_trans -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlPreFinishTransArg, - table_id_, - task_id_, - trans_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlPreFinishTransArg, + table_id_, + task_id_, + trans_id_); // confirm_finish_trans -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlConfirmFinishTransArg, - table_id_, - task_id_, - trans_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlConfirmFinishTransArg, + table_id_, + task_id_, + trans_id_); // abandon_trans -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlAbandonTransArg, - table_id_, - task_id_, - trans_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlAbandonTransArg, + table_id_, + task_id_, + trans_id_); // get_trans_status -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlGetTransStatusArg, - table_id_, - task_id_, - trans_id_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlGetTransStatusArg, + table_id_, + task_id_, + trans_id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlGetTransStatusRes, - trans_status_, - error_code_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlGetTransStatusRes, + trans_status_, + error_code_); // insert_trans -OB_SERIALIZE_MEMBER_SIMPLE(ObDirectLoadControlInsertTransArg, - table_id_, - task_id_, - trans_id_, - session_id_, - sequence_no_, - payload_); +OB_SERIALIZE_MEMBER(ObDirectLoadControlInsertTransArg, + table_id_, + task_id_, + trans_id_, + session_id_, + sequence_no_, + payload_); } // namespace observer } // namespace oceanbase diff --git a/src/observer/table_load/control/ob_table_load_control_rpc_struct.h b/src/observer/table_load/control/ob_table_load_control_rpc_struct.h index ebddf0ae6..f287eaf33 100644 --- a/src/observer/table_load/control/ob_table_load_control_rpc_struct.h +++ b/src/observer/table_load/control/ob_table_load_control_rpc_struct.h @@ -15,11 +15,13 @@ #include "lib/container/ob_array_serialization.h" #include "lib/utility/ob_print_utils.h" #include "lib/utility/ob_unify_serialize.h" +#include "observer/table_load/ob_table_load_struct.h" #include "share/table/ob_table_load_array.h" #include "share/table/ob_table_load_define.h" #include "share/table/ob_table_load_sql_statistics.h" #include "sql/session/ob_sql_session_mgr.h" #include "observer/table_load/ob_table_load_struct.h" +#include "storage/direct_load/ob_direct_load_struct.h" namespace oceanbase { @@ -51,7 +53,7 @@ enum class ObDirectLoadControlCommandType struct ObDirectLoadControlRequest { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObDirectLoadControlRequest() : command_type_(observer::ObDirectLoadControlCommandType::MAX_TYPE) @@ -96,7 +98,7 @@ public: class ObDirectLoadControlResult { - OB_UNIS_VERSION(1); + OB_UNIS_VERSION(2); public: ObDirectLoadControlResult() @@ -152,11 +154,22 @@ class ObDirectLoadControlPreBeginArg final public: ObDirectLoadControlPreBeginArg(); ~ObDirectLoadControlPreBeginArg(); - TO_STRING_KV(K_(table_id), K_(config), K_(column_count), K_(dup_action), K_(px_mode), - K_(online_opt_stat_gather), K_(dest_table_id), K_(task_id), K_(schema_version), - K_(snapshot_version), K_(data_version), K_(partition_id_array), - K_(target_partition_id_array), K_(avail_memory), K_(write_session_count), - K_(exe_mode), K_(cluster_version)); + TO_STRING_KV(K_(table_id), + K_(config), + K_(column_count), + K_(dup_action), + K_(px_mode), + K_(online_opt_stat_gather), + K_(ddl_param), + K_(partition_id_array), + K_(target_partition_id_array), + KP_(session_info), + K_(free_session_ctx), + K_(avail_memory), + K_(write_session_count), + K_(exe_mode), + "method", storage::ObDirectLoadMethod::get_type_string(method_), + "insert_mode", storage::ObDirectLoadInsertMode::get_type_string(insert_mode_)); public: uint64_t table_id_; @@ -165,12 +178,7 @@ public: sql::ObLoadDupActionType dup_action_; bool px_mode_; bool online_opt_stat_gather_; - // ddl param - uint64_t dest_table_id_; - int64_t task_id_; - int64_t schema_version_; - int64_t snapshot_version_; - int64_t data_version_; + ObTableLoadDDLParam ddl_param_; // partition info table::ObTableLoadArray partition_id_array_; // origin table table::ObTableLoadArray target_partition_id_array_; // target table @@ -179,7 +187,8 @@ public: int64_t avail_memory_; int32_t write_session_count_; ObTableLoadExeMode exe_mode_; - uint64_t cluster_version_; + storage::ObDirectLoadMethod::Type method_; + storage::ObDirectLoadInsertMode::Type insert_mode_; }; class ObDirectLoadControlConfirmBeginArg final diff --git a/src/observer/table_load/ob_table_load_client_task.cpp b/src/observer/table_load/ob_table_load_client_task.cpp index 086b936f9..da88f46fe 100644 --- a/src/observer/table_load/ob_table_load_client_task.cpp +++ b/src/observer/table_load/ob_table_load_client_task.cpp @@ -589,6 +589,8 @@ int ObTableLoadClientTask::init_instance() load_param.px_mode_ = false; load_param.online_opt_stat_gather_ = false; // 支持统计信息收集需要构造ObExecContext load_param.dup_action_ = param_.get_dup_action(); + load_param.method_ = ObDirectLoadMethod::FULL; + load_param.insert_mode_ = ObDirectLoadInsertMode::NORMAL; const ObTableLoadTableCtx *tmp_ctx = nullptr; if (OB_FAIL(instance_.init(load_param, column_idxs, exec_ctx_))) { LOG_WARN("fail to init instance", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index c39b804ef..75ac63ce8 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -28,6 +28,7 @@ #include "observer/table_load/ob_table_load_utils.h" #include "share/ob_share_util.h" #include "share/stat/ob_incremental_stat_estimator.h" +#include "observer/omt/ob_tenant.h" namespace oceanbase { @@ -484,15 +485,12 @@ int ObTableLoadCoordinator::pre_begin_peers(ObDirectLoadResourceApplyArg &apply_ arg.dup_action_ = param_.dup_action_; arg.px_mode_ = param_.px_mode_; arg.online_opt_stat_gather_ = param_.online_opt_stat_gather_; - arg.dest_table_id_ = ctx_->ddl_param_.dest_table_id_; - arg.task_id_ = ctx_->ddl_param_.task_id_; - arg.schema_version_ = ctx_->ddl_param_.schema_version_; - arg.snapshot_version_ = ctx_->ddl_param_.snapshot_version_; - arg.data_version_ = ctx_->ddl_param_.data_version_; - arg.cluster_version_ = ctx_->ddl_param_.cluster_version_; + arg.ddl_param_ = ctx_->ddl_param_; arg.session_info_ = ctx_->session_info_; arg.write_session_count_ = param_.write_session_count_; arg.exe_mode_ = ctx_->param_.exe_mode_; + arg.method_ = param_.method_; + arg.insert_mode_ = param_.insert_mode_; for (int64_t i = 0; OB_SUCC(ret) && i < all_leader_info_array.count(); ++i) { const ObTableLoadPartitionLocation::LeaderInfo &leader_info = all_leader_info_array.at(i); const ObTableLoadPartitionLocation::LeaderInfo &target_leader_info = diff --git a/src/observer/table_load/ob_table_load_instance.cpp b/src/observer/table_load/ob_table_load_instance.cpp index 09a961345..e8de04a11 100644 --- a/src/observer/table_load/ob_table_load_instance.cpp +++ b/src/observer/table_load/ob_table_load_instance.cpp @@ -81,8 +81,10 @@ int ObTableLoadInstance::init(ObTableLoadParam ¶m, const ObIArray & LOG_WARN("fail to check tenant", KR(ret), K(param.tenant_id_)); } // check support - else if (OB_FAIL(ObTableLoadService::check_support_direct_load(param.table_id_))) { - LOG_WARN("fail to check support direct load", KR(ret), K(param.table_id_)); + else if (OB_FAIL(ObTableLoadService::check_support_direct_load(param.table_id_, + param.method_, + param.insert_mode_))) { + LOG_WARN("fail to check support direct load", KR(ret), K(param)); } // create table ctx else if (OB_FAIL(create_table_ctx(param, idx_array))) { diff --git a/src/observer/table_load/ob_table_load_merger.cpp b/src/observer/table_load/ob_table_load_merger.cpp index d40ccb24d..ad33df363 100644 --- a/src/observer/table_load/ob_table_load_merger.cpp +++ b/src/observer/table_load/ob_table_load_merger.cpp @@ -284,6 +284,7 @@ int ObTableLoadMerger::build_merge_ctx() merge_param.is_column_store_ = store_ctx_->ctx_->schema_.is_column_store_; merge_param.fill_cg_thread_cnt_ = param_.session_count_; merge_param.px_mode_ = param_.px_mode_; + merge_param.insert_mode_ = param_.insert_mode_; merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_; merge_param.dml_row_handler_ = store_ctx_->error_row_handler_; if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_ctx_->ls_partition_ids_, diff --git a/src/observer/table_load/ob_table_load_redef_table.cpp b/src/observer/table_load/ob_table_load_redef_table.cpp index 21bb1e147..91d5e9482 100644 --- a/src/observer/table_load/ob_table_load_redef_table.cpp +++ b/src/observer/table_load/ob_table_load_redef_table.cpp @@ -34,6 +34,18 @@ int ObTableLoadRedefTable::start(const ObTableLoadRedefTableStartArg &arg, if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg)); + } else if (session_info.get_ddl_info().is_mview_complete_refresh()) { + res.task_id_ = session_info.get_cur_exec_ctx()->get_table_direct_insert_ctx().get_ddl_task_id(); + share::ObDDLTaskStatus status = share::ObDDLTaskStatus::PREPARE; + if (OB_FAIL(ObDDLUtil::get_data_information(arg.tenant_id_, + res.task_id_, + res.data_format_version_, + res.snapshot_version_, + status, + res.dest_table_id_, + res.schema_version_))) { + LOG_WARN("fail to get ddl task info", KR(ret), K(arg)); + } } else { const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); ObCreateHiddenTableArg create_table_arg; @@ -78,6 +90,8 @@ int ObTableLoadRedefTable::finish(const ObTableLoadRedefTableFinishArg &arg, if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg)); + } else if (session_info.get_ddl_info().is_mview_complete_refresh()) { + //pass } else { const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); ObCopyTableDependentsArg copy_table_dependents_arg; @@ -131,6 +145,8 @@ int ObTableLoadRedefTable::abort(const ObTableLoadRedefTableAbortArg &arg, if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(arg)); + } else if (session_info.get_ddl_info().is_mview_complete_refresh()) { + //pass } else { const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); ObAbortRedefTableArg abort_redef_table_arg; diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 35a4b35c6..e63666bc4 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -393,12 +393,17 @@ int ObTableLoadService::check_tenant() return ret; } -int ObTableLoadService::check_support_direct_load(uint64_t table_id) +int ObTableLoadService::check_support_direct_load( + const uint64_t table_id, + const ObDirectLoadMethod::Type method, + const ObDirectLoadInsertMode::Type insert_mode) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_ID == table_id)) { + if (OB_UNLIKELY(OB_INVALID_ID == table_id || + !ObDirectLoadMethod::is_type_valid(method) || + !ObDirectLoadInsertMode::is_type_valid(insert_mode))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(table_id)); + LOG_WARN("invalid args", KR(ret), K(table_id), K(method), K(insert_mode)); } else { const uint64_t tenant_id = MTL_ID(); ObSchemaGetterGuard schema_guard; @@ -453,6 +458,15 @@ int ObTableLoadService::check_support_direct_load(uint64_t table_id) ret = OB_NOT_SUPPORTED; LOG_WARN("direct-load does not support table with materialized view log", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table with materialized view log"); + } else if (OB_UNLIKELY(ObDirectLoadMethod::is_incremental(method))) { // incremental direct-load + ret = OB_NOT_SUPPORTED; + LOG_WARN("incremental direct-load is not supported", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "incremental direct-load is not supported"); + } else if (ObDirectLoadMethod::is_full(method)) { // full direct-load + if (OB_UNLIKELY(!ObDirectLoadInsertMode::is_valid_for_full_method(insert_mode))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected insert mode for full direct-load", KR(ret), K(method), K(insert_mode)); + } } } return ret; diff --git a/src/observer/table_load/ob_table_load_service.h b/src/observer/table_load/ob_table_load_service.h index 45c7631a4..ee0ca6622 100644 --- a/src/observer/table_load/ob_table_load_service.h +++ b/src/observer/table_load/ob_table_load_service.h @@ -22,6 +22,7 @@ #include "observer/table_load/ob_table_load_assigned_memory_manager.h" #include "observer/table_load/resource/ob_table_load_resource_rpc_proxy.h" #include "observer/table_load/resource/ob_table_load_resource_service.h" +#include "storage/direct_load/ob_direct_load_struct.h" namespace oceanbase { @@ -34,7 +35,9 @@ class ObTableLoadService public: static int mtl_init(ObTableLoadService *&service); static int check_tenant(); - static int check_support_direct_load(uint64_t table_id); + static int check_support_direct_load(const uint64_t table_id, + const storage::ObDirectLoadMethod::Type method, + const storage::ObDirectLoadInsertMode::Type insert_mode); static ObTableLoadTableCtx *alloc_ctx(); static void free_ctx(ObTableLoadTableCtx *table_ctx); static int add_ctx(ObTableLoadTableCtx *table_ctx); diff --git a/src/observer/table_load/ob_table_load_struct.cpp b/src/observer/table_load/ob_table_load_struct.cpp index a9e7bb0a6..293f6d300 100644 --- a/src/observer/table_load/ob_table_load_struct.cpp +++ b/src/observer/table_load/ob_table_load_struct.cpp @@ -1,5 +1,5 @@ /** - * Copyright (c) 2023 OceanBase + * Copyright (c) 2024 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: @@ -9,12 +9,27 @@ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ + #define USING_LOG_PREFIX SERVER + #include "observer/table_load/ob_table_load_struct.h" - namespace oceanbase + +namespace oceanbase { namespace observer { +using namespace common; +using namespace table; + OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadUniqueKey, table_id_, task_id_); + +OB_SERIALIZE_MEMBER(ObTableLoadDDLParam, + dest_table_id_, + task_id_, + schema_version_, + snapshot_version_, + data_version_, + cluster_version_); + } // namespace observer -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_struct.h b/src/observer/table_load/ob_table_load_struct.h index 19849d533..8b64a887f 100644 --- a/src/observer/table_load/ob_table_load_struct.h +++ b/src/observer/table_load/ob_table_load_struct.h @@ -17,6 +17,7 @@ #include "lib/utility/ob_print_utils.h" #include "share/table/ob_table_load_array.h" #include "sql/resolver/cmd/ob_load_data_stmt.h" +#include "storage/direct_load/ob_direct_load_struct.h" namespace oceanbase { @@ -145,7 +146,9 @@ struct ObTableLoadParam dup_action_(sql::ObLoadDupActionType::LOAD_INVALID_MODE), avail_memory_(0), write_session_count_(0), - exe_mode_(ObTableLoadExeMode::MAX_TYPE) + exe_mode_(ObTableLoadExeMode::MAX_TYPE), + method_(storage::ObDirectLoadMethod::INVALID_METHOD), + insert_mode_(storage::ObDirectLoadInsertMode::INVALID_INSERT_MODE) { } @@ -167,12 +170,37 @@ struct ObTableLoadParam parallel_ > 0 && session_count_ > 0 && batch_size_ > 0 && - column_count_ > 0; + column_count_ > 0 && + sql::ObLoadDupActionType::LOAD_INVALID_MODE != dup_action_ && + storage::ObDirectLoadMethod::is_type_valid(method_) && + storage::ObDirectLoadInsertMode::is_type_valid(insert_mode_) && + (storage::ObDirectLoadMethod::is_full(method_) + ? storage::ObDirectLoadInsertMode::is_valid_for_full_method(insert_mode_) + : true) && + (storage::ObDirectLoadMethod::is_incremental(method_) + ? storage::ObDirectLoadInsertMode::is_valid_for_incremental_method(insert_mode_) + : true) && + (storage::ObDirectLoadInsertMode::INC_REPLACE == insert_mode_ + ? sql::ObLoadDupActionType::LOAD_REPLACE == dup_action_ + : true); } - TO_STRING_KV(K_(tenant_id), K_(table_id), K_(parallel), K_(session_count), K_(batch_size), - K_(max_error_row_count), K_(sql_mode), K_(column_count), K_(need_sort), K_(px_mode), - K_(online_opt_stat_gather), K_(dup_action), K_(avail_memory), K_(write_session_count), K_(exe_mode)); + TO_STRING_KV(K_(tenant_id), + K_(table_id), + K_(parallel), + K_(session_count), + K_(batch_size), + K_(max_error_row_count), + K_(column_count), + K_(need_sort), + K_(px_mode), + K_(online_opt_stat_gather), + K_(dup_action), + K_(avail_memory), + K_(write_session_count), + K_(exe_mode), + "method", storage::ObDirectLoadMethod::get_type_string(method_), + "insert_mode", storage::ObDirectLoadInsertMode::get_type_string(insert_mode_)); public: uint64_t tenant_id_; uint64_t table_id_; @@ -180,7 +208,7 @@ public: int32_t session_count_; int32_t batch_size_; uint64_t max_error_row_count_; - uint64_t sql_mode_; + uint64_t sql_mode_; // unused int32_t column_count_; bool need_sort_; bool px_mode_; @@ -189,10 +217,13 @@ public: int64_t avail_memory_; int32_t write_session_count_; ObTableLoadExeMode exe_mode_; + storage::ObDirectLoadMethod::Type method_; + storage::ObDirectLoadInsertMode::Type insert_mode_; }; struct ObTableLoadDDLParam { + OB_UNIS_VERSION(1); public: ObTableLoadDDLParam() : dest_table_id_(common::OB_INVALID_ID), diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp index a1dbb1329..5261109ca 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp @@ -191,8 +191,8 @@ int ObTableLoadTransBucketWriter::write(int32_t session_id, ObTableLoadObjRowArr } } else { if (coordinator_ctx_->partition_calc_.is_partition_with_autoinc_ && - OB_FAIL(handle_partition_with_autoinc_identity(session_ctx, obj_rows, param_.sql_mode_, - session_id))) { + OB_FAIL(handle_partition_with_autoinc_identity(session_ctx, obj_rows, + trans_ctx_->ctx_->session_info_->get_sql_mode(), session_id))) { LOG_WARN("fail to handle partition column with autoincrement or identity", KR(ret)); } else if (OB_FAIL(write_for_partitioned(session_ctx, obj_rows))) { LOG_WARN("fail to write for partitioned", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index 77ef0be4c..0eeb3fd45 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -442,7 +442,7 @@ int ObTableLoadTransStoreWriter::handle_autoinc_column(const ObColumnSchemaV2 *c int ret = OB_SUCCESS; if (OB_FAIL(ObTableLoadAutoincNextval::eval_nextval( &(store_ctx_->session_ctx_array_[session_id - 1].autoinc_param_), datum, tc, - param_.sql_mode_))) { + store_ctx_->ctx_->session_info_->get_sql_mode()))) { LOG_WARN("fail to get auto increment next value", KR(ret)); } return ret; diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index ccc555b71..ace6f776c 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -187,6 +187,7 @@ ob_set_subtarget(ob_rootserver mview mview/ob_mview_maintenance_task.cpp mview/ob_mview_refresh_stats_maintenance_task.cpp mview/ob_mview_timer_task.cpp + mview/ob_mview_dependency_service.cpp ) ob_server_add_target(ob_rootserver) diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index 010c28297..4b3718e9f 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -52,12 +52,13 @@ ObDDLRedefinitionSSTableBuildTask::ObDDLRedefinitionSSTableBuildTask( const int64_t mview_table_id, ObRootService *root_service, const common::ObAddr &inner_sql_exec_addr, - const int64_t data_format_version) + const int64_t data_format_version, + const bool is_retryable_ddl) : is_inited_(false), tenant_id_(tenant_id), task_id_(task_id), data_table_id_(data_table_id), dest_table_id_(dest_table_id), schema_version_(schema_version), snapshot_version_(snapshot_version), execution_id_(execution_id), consumer_group_id_(consumer_group_id), sql_mode_(sql_mode), trace_id_(trace_id), parallelism_(parallelism), use_heap_table_ddl_plan_(use_heap_table_ddl_plan), - is_mview_complete_refresh_(is_mview_complete_refresh), mview_table_id_(mview_table_id), + is_mview_complete_refresh_(is_mview_complete_refresh), is_retryable_ddl_(is_retryable_ddl), mview_table_id_(mview_table_id), root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr), data_format_version_(0) { set_retry_times(0); // do not retry @@ -130,7 +131,7 @@ int ObDDLRedefinitionSSTableBuildTask::process() if (is_mview_complete_refresh_) { if (OB_FAIL(ObDDLUtil::generate_build_mview_replica_sql(tenant_id_, mview_table_id_, - dest_table_id_, + data_table_id_, schema_guard, snapshot_version_, execution_id_, @@ -167,11 +168,12 @@ int ObDDLRedefinitionSSTableBuildTask::process() ObSessionParam session_param; session_param.sql_mode_ = reinterpret_cast(&sql_mode_); session_param.tz_info_wrap_ = &tz_info_wrap_; - session_param.ddl_info_.set_is_ddl(true); + session_param.ddl_info_.set_is_ddl(!is_mview_complete_refresh_); session_param.ddl_info_.set_source_table_hidden(false); session_param.ddl_info_.set_dest_table_hidden(true); session_param.ddl_info_.set_heap_table_ddl(use_heap_table_ddl_plan_); session_param.ddl_info_.set_mview_complete_refresh(is_mview_complete_refresh_); + session_param.ddl_info_.set_retryable_ddl(is_retryable_ddl_); session_param.use_external_session_ = true; // means session id dispatched by session mgr session_param.consumer_group_id_ = consumer_group_id_; @@ -200,7 +202,7 @@ int ObDDLRedefinitionSSTableBuildTask::process() oracle_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE, &session_param, sql_exec_addr))) { LOG_WARN("fail to execute build replica sql", K(ret), K(tenant_id_)); } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && !is_mview_complete_refresh_) { if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(tenant_id_, dest_table_id_, execution_id_, task_id_))) { LOG_WARN("fail to check sstable checksum_report_finish", K(ret), K(tenant_id_), K(dest_table_id_), K(execution_id_), K(task_id_)); @@ -257,7 +259,8 @@ ObAsyncTask *ObDDLRedefinitionSSTableBuildTask::deep_copy(char *buf, const int64 mview_table_id_, root_service_, inner_sql_exec_addr_, - data_format_version_); + data_format_version_, + is_retryable_ddl_); if (OB_FAIL(new_task->tz_info_wrap_.deep_copy(tz_info_wrap_))) { LOG_WARN("failed to copy tz info wrap", K(ret)); } else if (OB_FAIL(new_task->col_name_map_.assign(col_name_map_))) { diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h index faac17aa3..fb2477fc5 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h @@ -42,7 +42,8 @@ public: const int64_t mview_table_id, ObRootService *root_service, const common::ObAddr &inner_sql_exec_addr, - const int64_t data_format_version = 0); + const int64_t data_format_version, + const bool is_retryable_ddl); int init( const ObTableSchema &orig_table_schema, const ObTableSchema &hidden_table_schema, @@ -72,6 +73,7 @@ private: int64_t parallelism_; bool use_heap_table_ddl_plan_; bool is_mview_complete_refresh_; + bool is_retryable_ddl_; int64_t mview_table_id_; common::ObArray based_schema_object_infos_; ObRootService *root_service_; diff --git a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp index a499d6b56..7b84b8ea6 100755 --- a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp +++ b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp @@ -1025,6 +1025,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, case DDL_DIRECT_LOAD_INSERT: case DDL_ALTER_COLUMN_GROUP: case DDL_MVIEW_COMPLETE_REFRESH: + case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: if (OB_FAIL(create_table_redefinition_task(proxy, param.type_, param.src_table_schema_, @@ -2076,6 +2077,7 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record) case DDL_DIRECT_LOAD_INSERT: case DDL_ALTER_COLUMN_GROUP: case DDL_MVIEW_COMPLETE_REFRESH: + case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: ret = schedule_table_redefinition_task(record); break; case DDL_CREATE_MVIEW: @@ -2648,6 +2650,7 @@ int ObDDLScheduler::on_sstable_complement_job_reply( case ObDDLType::DDL_TABLE_REDEFINITION: case ObDDLType::DDL_ALTER_COLUMN_GROUP: case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: + case ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: if (OB_FAIL(static_cast(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) { LOG_WARN("update complete sstable job status", K(ret)); } @@ -2781,6 +2784,7 @@ int ObDDLScheduler::notify_update_autoinc_end(const ObDDLTaskKey &task_key, case ObDDLType::DDL_DIRECT_LOAD_INSERT: case ObDDLType::DDL_ALTER_COLUMN_GROUP: case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: + case ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: if (OB_FAIL(static_cast(&task)->notify_update_autoinc_finish(autoinc_val, ret_code))) { LOG_WARN("update complete sstable job status", K(ret)); } diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index 1b21db361..b3508ab36 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -814,6 +814,9 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st case DDL_MANUAL_SPLIT_NON_RANGE: ddl_type_str = "manual split non range"; break; + case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION: + ddl_type_str = "modify auto increment column with redefinition"; + break; default: ret = OB_ERR_UNEXPECTED; } @@ -980,6 +983,7 @@ int ObDDLTask::convert_to_record( task_record.task_version_ = get_task_version(); task_record.execution_id_ = get_execution_id(); task_record.ret_code_ = get_ret_code(); + task_record.ddl_need_retry_at_executor_ = !task_can_retry(); const ObString &ddl_stmt_str = get_ddl_stmt_str(); if (serialize_param_size > 0) { char *buf = nullptr; @@ -1413,7 +1417,7 @@ int64_t ObDDLTask::get_execution_id() const return execution_id_; } -int ObDDLTask::push_execution_id(const uint64_t tenant_id, const int64_t task_id, int64_t &new_execution_id) +int ObDDLTask::push_execution_id(const uint64_t tenant_id, const int64_t task_id, const bool ddl_can_retry, const int64_t data_format_version, int64_t &new_execution_id) { int ret = OB_SUCCESS; ObMySQLTransaction trans; @@ -1429,8 +1433,29 @@ int ObDDLTask::push_execution_id(const uint64_t tenant_id, const int64_t task_id } else { if (OB_FAIL(ObDDLTaskRecordOperator::select_for_update(trans, tenant_id, task_id, task_status, execution_id, ret_code))) { LOG_WARN("select for update failed", K(ret), K(task_id)); - } else if (OB_FAIL(ObDDLTaskRecordOperator::update_execution_id(trans, tenant_id, task_id, execution_id + 1))) { - LOG_WARN("update task status failed", K(ret)); + } else { + LOG_INFO("push execution id", K(tenant_id), K(task_id), K(task_status), K(execution_id), K(ret_code)); + if (ObDDLUtil::use_idempotent_mode(data_format_version)) { + if (0 == execution_id) { + // has been executed before + if (!ddl_can_retry) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("do not retry for heap table ddl plan", K(tenant_id), K(task_id), K(ddl_can_retry)); + } else { + if (OB_FAIL(ObDDLTaskRecordOperator::update_execution_id(trans, tenant_id, task_id, 0L/*execution id*/))) { + LOG_WARN("update task status failed", K(ret)); + } else { + new_execution_id = 0L; + } + } + } + } else { + if (OB_FAIL(ObDDLTaskRecordOperator::update_execution_id(trans, tenant_id, task_id, execution_id + 1))) { + LOG_WARN("update task status failed", K(ret)); + } else { + new_execution_id = execution_id + 1; + } + } } bool commit = (OB_SUCCESS == ret); int tmp_ret = trans.end(commit); @@ -1438,9 +1463,6 @@ int ObDDLTask::push_execution_id(const uint64_t tenant_id, const int64_t task_id LOG_WARN("fail to end trans", K(tmp_ret)); ret = (OB_SUCCESS == ret) ? tmp_ret : ret; } - if (OB_SUCC(ret)) { - new_execution_id = execution_id + 1; - } } return ret; } diff --git a/src/rootserver/ddl_task/ob_ddl_task.h b/src/rootserver/ddl_task/ob_ddl_task.h index 0602b9f0d..c73b7b0b3 100755 --- a/src/rootserver/ddl_task/ob_ddl_task.h +++ b/src/rootserver/ddl_task/ob_ddl_task.h @@ -75,7 +75,8 @@ public: bool is_valid() const; void reset(); TO_STRING_KV(K_(task_id), K_(parent_task_id), K_(ddl_type), K_(trace_id), K_(task_status), K_(tenant_id), K_(object_id), - K_(schema_version), K_(target_object_id), K_(snapshot_version), K_(message), K_(task_version), K_(ret_code), K_(execution_id)); + K_(schema_version), K_(target_object_id), K_(snapshot_version), K_(message), K_(task_version), K_(ret_code), K_(execution_id), + K_(ddl_need_retry_at_executor)); public: static const int64_t MAX_MESSAGE_LENGTH = 4096; typedef common::ObFixedLengthString TaskMessage; @@ -96,6 +97,7 @@ public: int64_t ret_code_; int64_t execution_id_; ObString ddl_stmt_str_; + bool ddl_need_retry_at_executor_; }; struct ObDDLTaskInfo final @@ -557,7 +559,12 @@ public: bool need_schedule() { return next_schedule_ts_ <= ObTimeUtility::current_time(); } bool is_replica_build_need_retry(const int ret_code); int64_t get_execution_id() const; - static int push_execution_id(const uint64_t tenant_id, const int64_t task_id, int64_t &new_execution_id); + static int push_execution_id( + const uint64_t tenant_id, + const int64_t task_id, + const bool ddl_can_retry, + const int64_t data_format_version, + int64_t &new_execution_id); void check_ddl_task_execute_too_long(); static bool check_is_load_data(share::ObDDLType task_type); virtual bool support_longops_monitoring() const { return false; } @@ -608,10 +615,11 @@ protected: int copy_longops_stat(share::ObLongopsValue &value); virtual bool is_error_need_retry(const int ret_code) { - return !share::ObIDDLTask::in_ddl_retry_black_list(ret_code) && (share::ObIDDLTask::in_ddl_retry_white_list(ret_code) - || MAX_ERR_TOLERANCE_CNT > ++err_code_occurence_cnt_); + return task_can_retry() && (!share::ObIDDLTask::in_ddl_retry_black_list(ret_code) && (share::ObIDDLTask::in_ddl_retry_white_list(ret_code) + || MAX_ERR_TOLERANCE_CNT > ++err_code_occurence_cnt_)); } int init_ddl_task_monitor_info(const uint64_t target_table_id); + virtual bool task_can_retry() const { return true; } protected: static const int64_t TASK_EXECUTE_TIME_THRESHOLD = 3 * 24 * 60 * 60 * 1000000L; // 3 days common::TCRWLock lock_; diff --git a/src/rootserver/ddl_task/ob_index_build_task.cpp b/src/rootserver/ddl_task/ob_index_build_task.cpp index 6ec590233..9fd0bd526 100755 --- a/src/rootserver/ddl_task/ob_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_index_build_task.cpp @@ -853,7 +853,7 @@ int ObIndexBuildTask::send_build_single_replica_request() LOG_WARN("ObIndexBuildTask has not been inited", K(ret)); } else if (OB_FAIL(DDL_SIM(tenant_id_, task_id_, DDL_TASK_SEND_BUILD_REPLICA_REQUEST_FAILED))) { LOG_WARN("ddl sim failure", K(ret), K(tenant_id_), K(task_id_)); - } else if (OB_FAIL(ObDDLTask::push_execution_id(tenant_id_, task_id_, new_execution_id))) { + } else if (OB_FAIL(ObDDLTask::push_execution_id(tenant_id_, task_id_, true/*is ddl retryable*/, data_format_version_, new_execution_id))) { LOG_WARN("failed to fetch new execution id", K(ret)); } else { if (OB_FAIL(ObDDLUtil::get_sys_ls_leader_addr(GCONF.cluster_id, tenant_id_, create_index_arg_.inner_sql_exec_addr_))) { diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp index 770b44120..b04865d7c 100755 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp @@ -39,7 +39,8 @@ ObTableRedefinitionTask::ObTableRedefinitionTask() has_rebuild_index_(false), has_rebuild_constraint_(false), has_rebuild_foreign_key_(false), allocator_(lib::ObLabel("RedefTask")), is_copy_indexes_(true), is_copy_triggers_(true), is_copy_constraints_(true), is_copy_foreign_keys_(true), - is_ignore_errors_(false), is_do_finish_(false), target_cg_cnt_(0) + is_ignore_errors_(false), is_do_finish_(false), target_cg_cnt_(0), use_heap_table_ddl_plan_(false), + is_ddl_retryable_(true) { } @@ -114,6 +115,8 @@ int ObTableRedefinitionTask::init(const ObTableSchema* src_table_schema, LOG_WARN("fail to get target cg cnt", K(ret), KPC(dst_table_schema)); } else if (OB_FAIL(init_ddl_task_monitor_info(target_object_id_))) { LOG_WARN("init ddl task monitor info failed", K(ret)); + } else if (OB_FAIL(check_ddl_can_retry(dst_table_schema))) { + LOG_WARN("check use heap table ddl plan failed", K(ret)); } else { is_inited_ = true; ddl_tracing_.open(); @@ -259,7 +262,6 @@ int ObTableRedefinitionTask::send_build_replica_request_by_sql() { int ret = OB_SUCCESS; bool modify_autoinc = false; - bool use_heap_table_ddl_plan = false; ObRootService *root_service = GCTX.root_service_; int64_t new_execution_id = 0; if (OB_ISNULL(root_service)) { @@ -269,9 +271,7 @@ int ObTableRedefinitionTask::send_build_replica_request_by_sql() LOG_WARN("ddl sim failure", K(tenant_id_), K(task_id_)); } else if (OB_FAIL(check_modify_autoinc(modify_autoinc))) { LOG_WARN("failed to check modify autoinc", K(ret)); - } else if (OB_FAIL(check_use_heap_table_ddl_plan(use_heap_table_ddl_plan))) { - LOG_WARN("fail to check heap table ddl plan", K(ret)); - } else if (OB_FAIL(ObDDLTask::push_execution_id(tenant_id_, task_id_, new_execution_id))) { + } else if (OB_FAIL(ObDDLTask::push_execution_id(tenant_id_, task_id_, is_ddl_retryable_, data_format_version_, new_execution_id))) { LOG_WARN("failed to fetch new execution id", K(ret)); } else { ObSQLMode sql_mode = alter_table_arg_.sql_mode_; @@ -300,12 +300,13 @@ int ObTableRedefinitionTask::send_build_replica_request_by_sql() sql_mode, trace_id_, parallelism_, - use_heap_table_ddl_plan, + use_heap_table_ddl_plan_, alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_, alter_table_arg_.mview_refresh_info_.mview_table_id_, GCTX.root_service_, alter_table_arg_.inner_sql_exec_addr_, - data_format_version_); + data_format_version_, + is_ddl_retryable_); if (OB_FAIL(root_service->get_ddl_service().get_tenant_schema_guard_with_version_in_inner_table(tenant_id_, schema_guard))) { LOG_WARN("get schema guard failed", K(ret)); } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, object_id_, orig_table_schema))) { @@ -331,7 +332,7 @@ int ObTableRedefinitionTask::check_build_replica_end(bool &is_end) ret_code_ = complete_sstable_job_ret_code_; is_end = true; LOG_WARN("complete sstable job failed", K(ret_code_), K(object_id_), K(target_object_id_)); - if (is_replica_build_need_retry(ret_code_)) { + if (is_replica_build_need_retry(ret_code_) && is_ddl_retryable_) { build_replica_request_time_ = 0; complete_sstable_job_ret_code_ = INT64_MAX; ret_code_ = OB_SUCCESS; @@ -345,30 +346,42 @@ int ObTableRedefinitionTask::check_build_replica_end(bool &is_end) return ret; } -int ObTableRedefinitionTask::check_use_heap_table_ddl_plan(bool &use_heap_table_ddl_plan) +int ObTableRedefinitionTask::check_ddl_can_retry(const ObTableSchema *table_schema) { int ret = OB_SUCCESS; - use_heap_table_ddl_plan = false; - ObSchemaGetterGuard schema_guard; - const ObTableSchema *target_table_schema = nullptr; - ObRootService *root_service = GCTX.root_service_; - if (OB_ISNULL(root_service)) { - ret = OB_ERR_SYS; - LOG_WARN("error sys, root service must not be nullptr", K(ret)); + is_ddl_retryable_ = true; + if (OB_ISNULL(table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(table_schema)); + } else if (OB_FAIL(check_use_heap_table_ddl_plan(table_schema))) { + LOG_WARN("check use heap table ddl plan failed", K(ret)); + } else if (DDL_MVIEW_COMPLETE_REFRESH == task_type_) { + is_ddl_retryable_ = false; + } else { + if (ObDDLUtil::use_idempotent_mode(data_format_version_)) { + if (use_heap_table_ddl_plan_) { + is_ddl_retryable_ = false; + } else if (DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION == task_type_) { + is_ddl_retryable_ = false; + } + } + } + return ret; +} + +int ObTableRedefinitionTask::check_use_heap_table_ddl_plan(const ObTableSchema *target_table_schema) +{ + int ret = OB_SUCCESS; + use_heap_table_ddl_plan_ = false; + if (OB_ISNULL(target_table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(target_table_schema)); } else if (OB_FAIL(DDL_SIM(tenant_id_, task_id_, TABLE_REDEF_TASK_CHECK_USE_HEAP_PLAN_FAILED))) { LOG_WARN("ddl sim failure", K(tenant_id_), K(task_id_)); - } else if (OB_FAIL(root_service->get_ddl_service() - .get_tenant_schema_guard_with_version_in_inner_table(dst_tenant_id_, schema_guard))) { - LOG_WARN("get schema guard failed", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schema(dst_tenant_id_, target_object_id_, target_table_schema))) { - LOG_WARN("fail to get table schema", K(ret), K(target_object_id_)); - } else if (OB_ISNULL(target_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, table schema must not be nullptr", K(ret), K(target_object_id_)); } else if (target_table_schema->is_heap_table() && (DDL_ALTER_PARTITION_BY == task_type_ || DDL_DROP_PRIMARY_KEY == task_type_ || DDL_MVIEW_COMPLETE_REFRESH == task_type_)) { - use_heap_table_ddl_plan = true; + use_heap_table_ddl_plan_ = true; } return ret; } @@ -838,7 +851,6 @@ int ObTableRedefinitionTask::take_effect(const ObDDLTaskStatus next_task_status) ObRootService *root_service = GCTX.root_service_; ObSchemaGetterGuard schema_guard; const ObTableSchema *table_schema = nullptr; - bool use_heap_table_ddl_plan = false; ObDDLTaskStatus new_status = next_task_status; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -857,9 +869,7 @@ int ObTableRedefinitionTask::take_effect(const ObDDLTaskStatus next_task_status) LOG_WARN("table schema not exist", K(ret), K(target_object_id_)); } else if (!table_schema->is_user_hidden_table()) { LOG_INFO("target schema took effect", K(target_object_id_)); - } else if (OB_FAIL(check_use_heap_table_ddl_plan(use_heap_table_ddl_plan))) { - LOG_WARN("fail to check heap table ddl plan", K(ret)); - } else if (table_schema->is_heap_table() && !use_heap_table_ddl_plan && OB_FAIL(sync_tablet_autoinc_seq())) { + } else if (table_schema->is_heap_table() && !use_heap_table_ddl_plan_ && OB_FAIL(sync_tablet_autoinc_seq())) { if (OB_TIMEOUT == ret || OB_NOT_MASTER == ret) { ret = OB_SUCCESS; new_status = ObDDLTaskStatus::TAKE_EFFECT; @@ -1064,7 +1074,9 @@ int64_t ObTableRedefinitionTask::get_serialize_param_size() const + serialization::encoded_length_i8(copy_constraints) + serialization::encoded_length_i8(copy_foreign_keys) + serialization::encoded_length_i8(ignore_errors) + serialization::encoded_length_i8(do_finish) + serialization::encoded_length_i64(target_cg_cnt_) - + serialization::encoded_length_i64(complete_sstable_job_ret_code_); + + serialization::encoded_length_i64(complete_sstable_job_ret_code_) + + serialization::encoded_length_i8(use_heap_table_ddl_plan_) + + serialization::encoded_length_i8(is_ddl_retryable_); } int ObTableRedefinitionTask::serialize_params_to_message(char *buf, const int64_t buf_len, int64_t &pos) const @@ -1099,6 +1111,10 @@ int ObTableRedefinitionTask::serialize_params_to_message(char *buf, const int64_ LOG_WARN("fail to serialize target_cg_cnt", K(ret)); } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, complete_sstable_job_ret_code_))) { LOG_WARN("fail to serialize complete sstable job ret code", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, buf_len, pos, use_heap_table_ddl_plan_))) { + LOG_WARN("fail to serialize use heap table ddl plan", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, buf_len, pos, is_ddl_retryable_))) { + LOG_WARN("fail to serialize ddl can retry", K(ret)); } FLOG_INFO("serialize message for table redefinition", K(ret), K(copy_indexes), K(copy_triggers), K(copy_constraints), K(copy_foreign_keys), K(ignore_errors), K(do_finish), K(*this)); @@ -1152,7 +1168,23 @@ int ObTableRedefinitionTask::deserlize_params_from_message(const uint64_t tenant } if (OB_SUCC(ret) && pos < data_len) { if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &complete_sstable_job_ret_code_))) { - LOG_WARN("fail to deserialize is_do_finish_", K(ret)); + LOG_WARN("fail to deserialize complete sstable job ret code", K(ret)); + } + } + if (OB_SUCC(ret) && pos < data_len) { + int8_t use_heap_table_ddl_plan = false; + if (OB_FAIL(serialization::decode_i8(buf, data_len, pos, &use_heap_table_ddl_plan))) { + LOG_WARN("fail to deserialize use heap table ddl plan", K(ret)); + } else { + use_heap_table_ddl_plan_ = use_heap_table_ddl_plan; + } + } + if (OB_SUCC(ret) && pos < data_len) { + int8_t ddl_can_retry = false; + if (OB_FAIL(serialization::decode_i8(buf, data_len, pos, &ddl_can_retry))) { + LOG_WARN("fail to deserialize ddl can retry", K(ret)); + } else { + is_ddl_retryable_ = ddl_can_retry; } } } diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.h b/src/rootserver/ddl_task/ob_table_redefinition_task.h index 39bed2c5c..1b52cefc5 100644 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.h @@ -80,6 +80,7 @@ protected: const int64_t row_scanned, const int64_t row_inserted); int repending(const share::ObDDLTaskStatus next_task_status); + virtual bool task_can_retry() const override { return is_ddl_retryable_; } private: inline bool get_is_copy_indexes() const {return is_copy_indexes_;} inline bool get_is_copy_triggers() const {return is_copy_triggers_;} @@ -95,9 +96,10 @@ private: int check_build_replica_end(bool &is_end); int replica_end_check(const int ret_code); int check_modify_autoinc(bool &modify_autoinc); - int check_use_heap_table_ddl_plan(bool &use_heap_table_ddl_plan); + int check_use_heap_table_ddl_plan(const share::schema::ObTableSchema *target_table_schema); int get_direct_load_job_stat(common::ObArenaAllocator &allocator, sql::ObLoadDataStat &job_stat); int check_target_cg_cnt(); + int check_ddl_can_retry(const share::schema::ObTableSchema *table_schema); private: static const int64_t OB_TABLE_REDEFINITION_TASK_VERSION = 1L; bool has_rebuild_index_; @@ -111,6 +113,8 @@ private: bool is_ignore_errors_; bool is_do_finish_; int64_t target_cg_cnt_; + bool use_heap_table_ddl_plan_; + bool is_ddl_retryable_; }; } // end namespace rootserver diff --git a/src/rootserver/mview/ob_mview_dependency_service.cpp b/src/rootserver/mview/ob_mview_dependency_service.cpp new file mode 100644 index 000000000..c617c041a --- /dev/null +++ b/src/rootserver/mview/ob_mview_dependency_service.cpp @@ -0,0 +1,239 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS +#include "rootserver/mview/ob_mview_dependency_service.h" +#include "share/schema/ob_multi_version_schema_service.h" +#include "share/schema/ob_schema_getter_guard.h" +#include "share/schema/ob_table_sql_service.h" +#include "sql/resolver/mv/ob_mv_dep_utils.h" + +namespace oceanbase +{ +using namespace share; +using namespace share::schema; +using namespace sql; +namespace rootserver +{ +ObMViewDependencyService::ObMViewDependencyService(ObMultiVersionSchemaService &schema_service) + : schema_service_(schema_service) +{ + +} + +ObMViewDependencyService::~ObMViewDependencyService() +{ + +} + +int ObMViewDependencyService::remove_mview_dep_infos( + common::ObMySQLTransaction &trans, + share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const uint64_t mview_table_id) +{ + int ret = OB_SUCCESS; + ObArray stale_ref_table_ids; + // during upgrading, dropping mview should still work, + // hence, do nothing if __all_mview_dep does not exists + bool all_mview_dep_table_exists = false; + if (OB_FAIL(share::schema::ObSchemaUtils::check_sys_table_exist_by_sql( + trans, tenant_id, OB_ALL_MVIEW_DEP_TID, all_mview_dep_table_exists))) { + LOG_WARN("failed to check whether __all_mview_dep table exists", KR(ret)); + } else if (all_mview_dep_table_exists) { + if (OB_FAIL(ObMVDepUtils::get_table_ids_only_referenced_by_given_mv( + trans, tenant_id, mview_table_id, stale_ref_table_ids))) { + LOG_WARN("failed to get table ids only referenced by given mv", KR(ret)); + } else if (!stale_ref_table_ids.empty()) { + enum ObTableReferencedByMVFlag table_ref_by_mv_flag = + ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV; + if (OB_FAIL(update_mview_reference_table_status(trans, + schema_guard, + tenant_id, + stale_ref_table_ids, + table_ref_by_mv_flag))) { + LOG_WARN("failed to update mview reference table status", KR(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(sql::ObMVDepUtils::delete_mview_dep_infos( + trans, tenant_id, mview_table_id))) { + LOG_WARN("failed to delete mview dep infos", KR(ret), K(mview_table_id)); + } + } + return ret; +} + +int ObMViewDependencyService::update_mview_dep_infos( + ObMySQLTransaction &trans, + ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const uint64_t mview_table_id, + const common::ObIArray &dep_infos) +{ + int ret = OB_SUCCESS; + ObArray cur_mv_dep_infos; + ObArray prev_mv_dep_infos; + ObArray new_ref_table_ids;// table_referenced_by_mv_flag will be set + ObArray stale_ref_table_ids; // table_referenced_by_mv_flag will be cleared + ObArray table_ids_only_ref_by_this_mv; + // during upgrading, creating mview should still work, + // hence, do nothing if __all_mview_dep does not exists + bool all_mview_dep_table_exists = false; + + if ((OB_INVALID_TENANT_ID == tenant_id) || (OB_INVALID_ID == mview_table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or mview_table_id", KR(ret), K(tenant_id), K(mview_table_id)); + } else if (OB_FAIL(share::schema::ObSchemaUtils::check_sys_table_exist_by_sql( + trans, tenant_id, OB_ALL_MVIEW_DEP_TID, all_mview_dep_table_exists))) { + LOG_WARN("failed to check is system table name", KR(ret)); + } else if (all_mview_dep_table_exists) { + if (OB_FAIL(sql::ObMVDepUtils::convert_to_mview_dep_infos(dep_infos, cur_mv_dep_infos))) { + LOG_WARN("failed to convert to mview dep infos", KR(ret)); + } else if (OB_FAIL(sql::ObMVDepUtils::get_mview_dep_infos( + trans, tenant_id, mview_table_id, prev_mv_dep_infos))) { + LOG_WARN("failed to get mview dep infos", KR(ret)); + } else if (OB_FAIL(ObMVDepUtils::get_table_ids_only_referenced_by_given_mv( + trans, tenant_id, mview_table_id, table_ids_only_ref_by_this_mv))) { + LOG_WARN("failed to get table ids only referenced by given mv", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < cur_mv_dep_infos.count()); ++i) { + const ObMVDepInfo &cur_mv_dep = cur_mv_dep_infos.at(i); + if (OB_FAIL(new_ref_table_ids.push_back(cur_mv_dep.p_obj_))) { + LOG_WARN("failed to add cur ref table id to array", KR(ret)); + } + } + } + + if (OB_FAIL(ret)) { + } else if (prev_mv_dep_infos.empty()) { // creating a new mview + } else if (OB_UNLIKELY(prev_mv_dep_infos.count() != cur_mv_dep_infos.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv dep infos count not match", + KR(ret), K(prev_mv_dep_infos), K(cur_mv_dep_infos)); + } else { // updating an existing mview + for (int64_t i = 0; OB_SUCC(ret) && (i < cur_mv_dep_infos.count()); ++i) { + const ObMVDepInfo &prev_mv_dep = prev_mv_dep_infos.at(i); + const ObMVDepInfo &cur_mv_dep = cur_mv_dep_infos.at(i); + if (OB_UNLIKELY(prev_mv_dep.p_order_ != cur_mv_dep.p_order_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("p_order is not match", + KR(ret), K(prev_mv_dep.p_order_), K(cur_mv_dep.p_order_)); + } else if (prev_mv_dep.p_obj_ != cur_mv_dep.p_obj_) { + const uint64_t old_ref_table_id = prev_mv_dep.p_obj_; + // if an old_ref_table_id exists in the cur_mv_dep_infos, + // then its table_referenced_by_mv_flag does not need to be cleared + if (has_exist_in_array(new_ref_table_ids, old_ref_table_id)) { + } else if (has_exist_in_array(table_ids_only_ref_by_this_mv, prev_mv_dep.p_obj_)) { + // only when an old_ref_table_id exists in the list of table_ids_only_ref_by_this_mv, + // its table_referenced_by_mv_flag needs to be cleared + if (OB_FAIL(stale_ref_table_ids.push_back(prev_mv_dep.p_obj_))) { + LOG_WARN("failed to add old ref table id to array", KR(ret), K(prev_mv_dep.p_obj_)); + } + } + } + } + } + + if (OB_SUCC(ret) && !stale_ref_table_ids.empty()) { + enum ObTableReferencedByMVFlag table_ref_by_mv_flag = + ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV; + if (OB_FAIL(update_mview_reference_table_status(trans, + schema_guard, + tenant_id, + stale_ref_table_ids, + table_ref_by_mv_flag))) { + LOG_WARN("failed to update mview reference table status", KR(ret)); + } + } + + if (OB_SUCC(ret) && !new_ref_table_ids.empty()) { + enum ObTableReferencedByMVFlag table_ref_by_mv_flag = + ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV; + if (OB_FAIL(sql::ObMVDepUtils::delete_mview_dep_infos( + trans, tenant_id, mview_table_id))) { + LOG_WARN("failed to delete mview dep infos", KR(ret), K(mview_table_id)); + } else if (OB_FAIL(sql::ObMVDepUtils::insert_mview_dep_infos( + trans, tenant_id, mview_table_id, cur_mv_dep_infos))) { + LOG_WARN("failed to insert mview dep infos", KR(ret), K(new_ref_table_ids)); + } else if (OB_FAIL(update_mview_reference_table_status(trans, + schema_guard, + tenant_id, + new_ref_table_ids, + table_ref_by_mv_flag))) { + LOG_WARN("failed to update mview reference table status", + KR(ret), K(table_ref_by_mv_flag )); + } + } + } + + return ret; +} + +int ObMViewDependencyService::update_mview_reference_table_status( + ObMySQLTransaction &trans, + ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const ObIArray &ref_table_ids, + enum ObTableReferencedByMVFlag table_flag) +{ + int ret = OB_SUCCESS; + uint64_t compat_version = 0; + if ((ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV != table_flag) + && (ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV != table_flag)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table_flag", KR(ret), K(table_flag)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("failed to get data version", KR(ret), K(tenant_id)); + } else if (compat_version < DATA_VERSION_4_3_1_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("version lower than 4.3.1.0 does not support this operation", KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "tenant's data version is below 4.3.1.0, update mview reference table status is "); + } else { + ObSchemaService *schema_service = schema_service_.get_schema_service(); + for (int64_t i = 0; OB_SUCC(ret) && (i < ref_table_ids.count()); ++i) { + int64_t new_schema_version = OB_INVALID_VERSION; + const uint64_t ref_table_id = ref_table_ids.at(i); + const ObTableSchema *ref_table_schema = NULL; + if (OB_FAIL(schema_guard.get_table_schema( + tenant_id, ref_table_id, ref_table_schema))) { + LOG_WARN("failed to get table schema", KR(ret), K(tenant_id), K(ref_table_id)); + } else if (OB_ISNULL(ref_table_schema)) { + // the reference table has already been dropped, ignore it + LOG_TRACE("ref table schema is null", KR(ret), K(tenant_id), K(ref_table_id)); + } else if (table_flag == ObTableMode::get_table_referenced_by_mv_flag( + ref_table_schema->get_table_mode())) { + // bypass + } else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) { + LOG_WARN("fail to gen new schema_version", KR(ret), K(tenant_id)); + } else { + SMART_VAR(ObTableSchema, new_ref_table_schema) { + if (OB_FAIL(new_ref_table_schema.assign(*ref_table_schema))) { + LOG_WARN("fail to assign ref table schema", KR(ret)); + } else { + new_ref_table_schema.set_table_id(ref_table_id); + new_ref_table_schema.set_table_referenced_by_mv(table_flag); + new_ref_table_schema.set_schema_version(new_schema_version); + if (OB_FAIL(schema_service->get_table_sql_service().update_mview_reference_table_status( + new_ref_table_schema, trans))) { + LOG_WARN("failed to update mview reference table status", + KR(ret), K(ref_table_id), K(table_flag)); + } + } + } + } + } + } + return ret; +} +} // end of sql +} // end of oceanbase diff --git a/src/rootserver/mview/ob_mview_dependency_service.h b/src/rootserver/mview/ob_mview_dependency_service.h new file mode 100644 index 000000000..f0b12d453 --- /dev/null +++ b/src/rootserver/mview/ob_mview_dependency_service.h @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_MVIEW_OB_MVIEW_DEPENDENCY_SERVICE_H_ +#define OCEANBASE_ROOTSERVER_MVIEW_OB_MVIEW_DEPENDENCY_SERVICE_H_ + +#include "lib/ob_define.h" +#include "lib/container/ob_array.h" +#include "share/schema/ob_table_schema.h" + +namespace oceanbase +{ +namespace common +{ + class ObMySQLTransaction; +} +namespace share +{ +namespace schema +{ + class ObSchemaGetterGuard; + class ObMultiVersionSchemaService; + class ObDependencyInfo; +} +} +namespace rootserver +{ +class ObMViewDependencyService +{ +public: + ObMViewDependencyService(share::schema::ObMultiVersionSchemaService &schema_service); + ~ObMViewDependencyService(); + int update_mview_dep_infos(common::ObMySQLTransaction &trans, + share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const uint64_t mview_table_id, + const common::ObIArray &dep_infos); + int remove_mview_dep_infos(common::ObMySQLTransaction &trans, + share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const uint64_t mview_table_id); + int update_mview_reference_table_status( + common::ObMySQLTransaction &trans, + share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const ObIArray &ref_table_ids, + enum share::schema::ObTableReferencedByMVFlag table_flags); +private: + share::schema::ObMultiVersionSchemaService &schema_service_; +}; +} // end of sql +} // end of oceanbase +#endif \ No newline at end of file diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index de307cbc4..38beeae38 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -121,7 +121,7 @@ #include "share/schema/ob_mview_info.h" #include "storage/mview/ob_mview_sched_job_utils.h" #include "rootserver/restore/ob_tenant_clone_util.h" - +#include "rootserver/mview/ob_mview_dependency_service.h" namespace oceanbase { @@ -4125,11 +4125,7 @@ int ObDDLService::check_alter_table_column(obrpc::ObAlterTableArg &alter_table_a } else if (orig_table_schema.get_autoinc_column_id() == 0) { if (orig_column_schema->is_nullable()) { // if the original table has null, we need to do double write to fill the nulls - if (ObDDLType::DDL_INVALID == ddl_type || ObDDLType::DDL_MODIFY_COLUMN == ddl_type) { - ddl_type = ObDDLType::DDL_MODIFY_COLUMN; - } else { - ddl_type = ObDDLType::DDL_TABLE_REDEFINITION; - } + ddl_type = ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION; } else { if (ObDDLType::DDL_INVALID == ddl_type) { ddl_type = ObDDLType::DDL_MODIFY_AUTO_INCREMENT; @@ -13298,6 +13294,7 @@ int ObDDLService::do_offline_ddl_in_trans(obrpc::ObAlterTableArg &alter_table_ar LOG_WARN("submit ddl task failed", K(ret)); } else { res.task_id_ = task_record.task_id_; + res.ddl_need_retry_at_executor_ = task_record.ddl_need_retry_at_executor_; } } else if (is_simple_table_long_running_ddl(ddl_type)) { ObCreateDDLTaskParam param(tenant_id, @@ -13317,6 +13314,7 @@ int ObDDLService::do_offline_ddl_in_trans(obrpc::ObAlterTableArg &alter_table_ar LOG_WARN("submit ddl task failed", K(ret)); } else { res.task_id_ = task_record.task_id_; + res.ddl_need_retry_at_executor_ = task_record.ddl_need_retry_at_executor_; } } else { ret = OB_ERR_UNEXPECTED; @@ -13394,6 +13392,7 @@ int ObDDLService::create_hidden_table( int64_t refreshed_schema_version = 0; new_table_schema.set_tenant_id(dest_tenant_id); new_table_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_OFFLINE_DDL); + new_table_schema.set_table_referenced_by_mv(ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV); if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) { LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id)); } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, refreshed_schema_version))) { @@ -18733,6 +18732,14 @@ int ObDDLService::swap_orig_and_hidden_table_state(obrpc::ObAlterTableArg &alter LOG_WARN("fail to insert schema object dependency", KR(ret), K(dep)); } } + + if (OB_SUCC(ret)) { + ObMViewDependencyService mv_dep_service(*schema_service_); + if (OB_FAIL(mv_dep_service.update_mview_dep_infos( + trans, schema_guard, tenant_id, mview_table_id, deps))) { + LOG_WARN("failed to update mview dep infos", KR(ret)); + } + } } // update mview_info if (OB_SUCC(ret)) { @@ -21276,6 +21283,15 @@ int ObDDLService::drop_table_in_trans( table_schema, MATERIALIZED_VIEW, false)))) { LOG_WARN("drop_aux_table_in_drop_table failed", KR(ret)); } + + if (OB_SUCC(ret)) { + const uint64_t mview_table_id = table_schema.get_table_id(); + ObMViewDependencyService mv_dep_service(*schema_service_); + if (OB_FAIL(mv_dep_service.remove_mview_dep_infos( + trans, schema_guard, tenant_id, mview_table_id))) { + LOG_WARN("failed to remove mview dep infos", KR(ret)); + } + } } else if (!table_schema.is_aux_table()) { if (OB_FAIL((drop_aux_table_in_drop_table(trans, ddl_operator, schema_guard, table_schema, USER_INDEX, to_recyclebin)))) { @@ -39018,6 +39034,5 @@ bool ObDDLService::need_modify_dep_obj_status(const obrpc::ObAlterTableArg &alte || (alter_table_arg.is_alter_options_ && alter_table_schema.alter_option_bitset_.has_member(ObAlterTableArg::TABLE_NAME))); } - } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/ob_mlog_builder.cpp b/src/rootserver/ob_mlog_builder.cpp index a4b087751..b0d36dac6 100644 --- a/src/rootserver/ob_mlog_builder.cpp +++ b/src/rootserver/ob_mlog_builder.cpp @@ -29,7 +29,8 @@ namespace rootserver { ObMLogBuilder::MLogColumnUtils::MLogColumnUtils() : mlog_table_column_array_(), - allocator_("MlogColUtil") + allocator_("MlogColUtil"), + rowkey_count_(0) { } @@ -85,11 +86,10 @@ int ObMLogBuilder::MLogColumnUtils::check_column_type( return ret; } -int ObMLogBuilder::MLogColumnUtils::add_special_columns( - ObTableSchema &mlog_schema) +int ObMLogBuilder::MLogColumnUtils::add_special_columns() { int ret = OB_SUCCESS; - if (OB_FAIL(add_pk_column(mlog_schema))) { + if (OB_FAIL(add_sequence_column())) { LOG_WARN("failed to add sequence column", KR(ret)); } else if (OB_FAIL(add_dmltype_column())) { LOG_WARN("failed to add dmltype column", KR(ret)); @@ -99,9 +99,9 @@ int ObMLogBuilder::MLogColumnUtils::add_special_columns( return ret; } -// sequence column is mlog's rowkey -int ObMLogBuilder::MLogColumnUtils::add_pk_column( - ObTableSchema &mlog_schema) +// sequence_no is part of mlog's rowkey +// its rowkey position will be set later +int ObMLogBuilder::MLogColumnUtils::add_sequence_column() { int ret = OB_SUCCESS; ObColumnSchemaV2 *rowkey_column = nullptr; @@ -111,17 +111,16 @@ int ObMLogBuilder::MLogColumnUtils::add_pk_column( rowkey_column->set_autoincrement(false); rowkey_column->set_is_hidden(false); rowkey_column->set_nullable(false); - rowkey_column->set_rowkey_position(1); + rowkey_column->set_rowkey_position(0); rowkey_column->set_order_in_rowkey(ObOrderType::ASC); rowkey_column->set_column_id(OB_MLOG_SEQ_NO_COLUMN_ID); rowkey_column->set_data_type(ObIntType); rowkey_column->set_charset_type(CHARSET_BINARY); rowkey_column->set_collation_type(CS_TYPE_BINARY); - mlog_schema.set_rowkey_column_num(1); if (OB_FAIL(rowkey_column->set_column_name(OB_MLOG_SEQ_NO_COLUMN_NAME))) { LOG_WARN("failed to set column name", KR(ret)); - } else if (OB_FAIL(mlog_schema.add_column(*rowkey_column))) { - LOG_WARN("failed to add rowkey column to mlog schema", KR(ret)); + } else if (OB_FAIL(mlog_table_column_array_.push_back(rowkey_column))) { + LOG_WARN("failed to push back column to mlog table column array", KR(ret), KPC(rowkey_column)); } } return ret; @@ -201,9 +200,9 @@ int ObMLogBuilder::MLogColumnUtils::add_base_table_pk_columns( LOG_WARN("failed to assign rowkey_column to ref_column", KR(ret), KPC(rowkey_column)); } else { + // preserve the rowkey position of the column ref_column->set_autoincrement(false); ref_column->set_is_hidden(false); - ref_column->set_rowkey_position(0); ref_column->set_index_position(0); ref_column->set_prev_column_id(UINT64_MAX); ref_column->set_next_column_id(UINT64_MAX); @@ -215,6 +214,8 @@ int ObMLogBuilder::MLogColumnUtils::add_base_table_pk_columns( } else if (OB_FAIL(mlog_table_column_array_.push_back(ref_column))) { LOG_WARN("failed to push back column to mlog table column array", KR(ret), KP(ref_column)); + } else { + ++rowkey_count_; } } } @@ -269,7 +270,7 @@ int ObMLogBuilder::MLogColumnUtils::add_base_table_columns( return ret; } -int ObMLogBuilder::MLogColumnUtils::add_base_table_part_key_columns( +int ObMLogBuilder::MLogColumnUtils::implicit_add_base_table_part_key_columns( const ObPartitionKeyInfo &part_key_info, ObRowDesc &row_desc, const ObTableSchema &base_table_schema) @@ -327,12 +328,12 @@ int ObMLogBuilder::MLogColumnUtils::add_base_table_part_key_columns( int ret = OB_SUCCESS; const ObPartitionKeyInfo &part_key_info = base_table_schema.get_partition_key_info(); const ObPartitionKeyInfo &sub_part_key_info = base_table_schema.get_subpartition_key_info(); - if (OB_FAIL(add_base_table_part_key_columns( + if (OB_FAIL(implicit_add_base_table_part_key_columns( part_key_info, row_desc, base_table_schema))) { - LOG_WARN("failed to add base table part key columns", KR(ret)); - } else if (OB_FAIL(add_base_table_part_key_columns( + LOG_WARN("failed to implicit add base table part key columns", KR(ret)); + } else if (OB_FAIL(implicit_add_base_table_part_key_columns( sub_part_key_info, row_desc, base_table_schema))) { - LOG_WARN("failed to add base table sub part key columns", KR(ret)); + LOG_WARN("failed to implicit add base table sub part key columns", KR(ret)); } return ret; } @@ -350,7 +351,6 @@ int ObMLogBuilder::MLogColumnUtils::construct_mlog_table_columns( ObTableSchema &mlog_schema) { int ret = OB_SUCCESS; - int64_t rowkey_pos = 1; // the first rowkey is seq_no // sort mlog_table_column_array first struct ColumnSchemaCmp { inline bool operator()(const ObColumnSchemaV2 *a, const ObColumnSchemaV2 *b) { @@ -361,8 +361,13 @@ int ObMLogBuilder::MLogColumnUtils::construct_mlog_table_columns( for (int64_t i = 0; OB_SUCC(ret) && (i < mlog_table_column_array_.count()); ++i) { ObColumnSchemaV2 *column = mlog_table_column_array_.at(i); - if (column->is_part_key_column() || column->is_subpart_key_column()) { - column->set_rowkey_position(++rowkey_pos); + // columns referencing base table primary keys have already been assigned rowkey position + if ((column->is_part_key_column() || column->is_subpart_key_column()) + && (0 == column->get_rowkey_position())) { + column->set_rowkey_position(++rowkey_count_); + } else if (OB_MLOG_SEQ_NO_COLUMN_ID == column->get_column_id()) { + // mlog seq_no column must be the last rowkey + column->set_rowkey_position(++rowkey_count_); } if (OB_FAIL(mlog_schema.add_column(*column))) { LOG_WARN("failed to add column to mlog schema", KR(ret), KPC(column)); @@ -686,50 +691,44 @@ int ObMLogBuilder::set_table_columns( LOG_WARN("ObMLogBuilder not init", KR(ret)); } else { HEAP_VAR(ObRowDesc, row_desc) { - if (OB_FAIL(mlog_column_utils_.add_special_columns(mlog_schema))) { + if (base_table_schema.is_heap_table()) { + if (create_mlog_arg.with_primary_key_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create mlog on heap table cannot use with primary key option", + KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_primary_key_)); + } else if (!create_mlog_arg.with_rowid_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create mlog on heap table should use with rowid option", + KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_rowid_)); + } + } else { + if (create_mlog_arg.with_rowid_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create mlog on non-heap table cannot use with rowid option", + KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_rowid_)); + } else if (!create_mlog_arg.with_primary_key_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create mlog on non-heap table should use with primary key option", + KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_primary_key_)); + } + } + + // the primary key of a mlog is consist of: + // the base table pk + the base table part key + the mlog sequence no + if (OB_FAIL(ret)) { + } else if (OB_FAIL(mlog_column_utils_.add_base_table_pk_columns( + row_desc, base_table_schema))) { + LOG_WARN("failed to add base table pk columns", KR(ret)); + } else if (OB_FAIL(mlog_column_utils_.add_base_table_columns( + create_mlog_arg, row_desc, base_table_schema))) { + LOG_WARN("failed to add base table columns", KR(ret)); + } else if (OB_FAIL(mlog_column_utils_.add_base_table_part_key_columns( + row_desc, base_table_schema))) { + LOG_WARN("failed to add base table part key columns", KR(ret)); + } else if (OB_FAIL(mlog_column_utils_.add_special_columns())) { LOG_WARN("failed to add special columns", KR(ret)); - } - - if (OB_SUCC(ret)) { - if (base_table_schema.is_heap_table()) { - if (create_mlog_arg.with_primary_key_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("create mlog on heap table cannot use with primary key option", - KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_primary_key_)); - } else if (!create_mlog_arg.with_rowid_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("create mlog on heap table should use with rowid option", - KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_rowid_)); - } - } else { - if (create_mlog_arg.with_rowid_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("create mlog on non-heap table cannot use with rowid option", - KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_rowid_)); - } else if (!create_mlog_arg.with_primary_key_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("create mlog on non-heap table should use with primary key option", - KR(ret), K(base_table_schema.is_heap_table()), K(create_mlog_arg.with_primary_key_)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(mlog_column_utils_.add_base_table_pk_columns( - row_desc, base_table_schema))) { - LOG_WARN("failed to add base table pk columns", KR(ret)); - } - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(mlog_column_utils_.add_base_table_columns( - create_mlog_arg, row_desc, base_table_schema))) { - LOG_WARN("failed to add base table columns", KR(ret)); - } else if (OB_FAIL(mlog_column_utils_.add_base_table_part_key_columns( - row_desc, base_table_schema))) { - LOG_WARN("failed to add base table part key columns", KR(ret)); - } else if (OB_FAIL(mlog_column_utils_.construct_mlog_table_columns(mlog_schema))) { - LOG_WARN("failed to construct mlog table columns", KR(ret)); - } + } else if (OB_FAIL(mlog_column_utils_.construct_mlog_table_columns(mlog_schema))) { + LOG_WARN("failed to construct mlog table columns", KR(ret)); } } } diff --git a/src/rootserver/ob_mlog_builder.h b/src/rootserver/ob_mlog_builder.h index ae6c0d63b..db11b3c1c 100644 --- a/src/rootserver/ob_mlog_builder.h +++ b/src/rootserver/ob_mlog_builder.h @@ -54,20 +54,22 @@ private: const share::schema::ObTableSchema &base_table_schema); int add_base_table_part_key_columns(common::ObRowDesc &row_desc, const share::schema::ObTableSchema &base_table_schema); - int add_special_columns(share::schema::ObTableSchema &mlog_schema); + int add_special_columns(); int construct_mlog_table_columns(share::schema::ObTableSchema &mlog_schema); private: - int add_pk_column(share::schema::ObTableSchema &mlog_schema); + int add_sequence_column(); int add_dmltype_column(); int add_old_new_column(); - int add_base_table_part_key_columns(const common::ObPartitionKeyInfo &part_key_info, - common::ObRowDesc &row_desc, - const share::schema::ObTableSchema &base_table_schema); + int implicit_add_base_table_part_key_columns( + const common::ObPartitionKeyInfo &part_key_info, + common::ObRowDesc &row_desc, + const share::schema::ObTableSchema &base_table_schema); int alloc_column(ObColumnSchemaV2 *&column); public: ObArray mlog_table_column_array_; private: ObArenaAllocator allocator_; + int64_t rowkey_count_; }; private: diff --git a/src/share/inner_table/ob_inner_table_schema.501_550.cpp b/src/share/inner_table/ob_inner_table_schema.501_550.cpp index 0f09a0802..d2554ef3a 100644 --- a/src/share/inner_table/ob_inner_table_schema.501_550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.501_550.cpp @@ -1035,6 +1035,203 @@ int ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_schema(ObTableSch return ret; } +int ObInnerTableSchema::all_mview_dep_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_DEP_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_DEP_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("mview_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("p_order", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("p_obj", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("p_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("qbcid", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("flags", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_MVIEW_DEP_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_MVIEW_DEP_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp index b4827512f..c4df94db7 100644 --- a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp @@ -700,6 +700,141 @@ int ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_aux_lob_meta_sche return ret; } +int ObInnerTableSchema::all_mview_dep_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_DEP_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_DEP_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_MVIEW_DEP_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_MVIEW_DEP_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp index 35848d29e..88999d4df 100644 --- a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp @@ -475,6 +475,96 @@ int ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_aux_lob_piece_sch return ret; } +int ObInnerTableSchema::all_mview_dep_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_MVIEW_DEP_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 1c170e78e..b4a163d14 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -585,6 +585,7 @@ public: static int all_column_privilege_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_schema(share::schema::ObTableSchema &table_schema); + static int all_mview_dep_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_all_table_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_table_column_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_table_index_schema(share::schema::ObTableSchema &table_schema); @@ -2469,6 +2470,7 @@ public: static int all_column_privilege_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_mview_dep_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_table_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_ddl_operation_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -2762,6 +2764,7 @@ public: static int all_column_privilege_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_mview_dep_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_ash_all_virtual_ash_i1_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_sql_plan_monitor_all_virtual_sql_plan_monitor_i1_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_sql_audit_all_virtual_sql_audit_i1_schema(share::schema::ObTableSchema &table_schema); @@ -3266,6 +3269,7 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_column_privilege_schema, ObInnerTableSchema::all_column_privilege_history_schema, ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_schema, + ObInnerTableSchema::all_mview_dep_schema, NULL,}; const schema_create_func virtual_table_schema_creators [] = { @@ -5344,6 +5348,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_COLUMN_PRIVILEGE_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID, + OB_ALL_MVIEW_DEP_TID, OB_TENANT_VIRTUAL_ALL_TABLE_TID, OB_TENANT_VIRTUAL_TABLE_COLUMN_TID, OB_TENANT_VIRTUAL_TABLE_INDEX_TID, @@ -7049,6 +7054,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TID, + OB_ALL_MVIEW_DEP_AUX_LOB_META_TID, OB_ALL_TABLE_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_AUX_LOB_PIECE_TID, OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TID, @@ -7317,7 +7323,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TID, - OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID, }; + OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID, + OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID, }; const uint64_t all_ora_mapping_virtual_table_org_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_TID, @@ -7895,6 +7902,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_COLUMN_PRIVILEGE_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME, + OB_ALL_MVIEW_DEP_TNAME, OB_TENANT_VIRTUAL_ALL_TABLE_TNAME, OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME, OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME, @@ -9600,6 +9608,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TNAME, + OB_ALL_MVIEW_DEP_AUX_LOB_META_TNAME, OB_ALL_TABLE_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_AUX_LOB_PIECE_TNAME, OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TNAME, @@ -9868,7 +9877,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TNAME, - OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME, }; + OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME, + OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TNAME, }; const uint64_t only_rs_vtables [] = { OB_ALL_VIRTUAL_CORE_META_TABLE_TID, @@ -12717,6 +12727,14 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_aux_lob_piece_schema }, + { + OB_ALL_MVIEW_DEP_TID, + OB_ALL_MVIEW_DEP_AUX_LOB_META_TID, + OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_mview_dep_aux_lob_meta_schema, + ObInnerTableSchema::all_mview_dep_aux_lob_piece_schema + }, + }; static inline bool get_sys_table_lob_aux_table_id(const uint64_t tid, uint64_t& meta_tid, uint64_t& piece_tid) diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index b8378d8bc..c1b659310 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -321,6 +321,7 @@ const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_TID = 502; // "__all_trusted_root const uint64_t OB_ALL_COLUMN_PRIVILEGE_TID = 505; // "__all_column_privilege" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_TID = 506; // "__all_column_privilege_history" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID = 507; // "__all_tenant_snapshot_ls_replica_history" +const uint64_t OB_ALL_MVIEW_DEP_TID = 518; // "__all_mview_dep" const uint64_t OB_TENANT_VIRTUAL_ALL_TABLE_TID = 10001; // "__tenant_virtual_all_table" const uint64_t OB_TENANT_VIRTUAL_TABLE_COLUMN_TID = 10002; // "__tenant_virtual_table_column" const uint64_t OB_TENANT_VIRTUAL_TABLE_INDEX_TID = 10003; // "__tenant_virtual_table_index" @@ -2205,6 +2206,7 @@ const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_META_TID = 50502; // "__a const uint64_t OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID = 50505; // "__all_column_privilege_aux_lob_meta" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TID = 50506; // "__all_column_privilege_history_aux_lob_meta" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TID = 50507; // "__all_tenant_snapshot_ls_replica_history_aux_lob_meta" +const uint64_t OB_ALL_MVIEW_DEP_AUX_LOB_META_TID = 50518; // "__all_mview_dep_aux_lob_meta" const uint64_t OB_ALL_TABLE_AUX_LOB_PIECE_TID = 60003; // "__all_table_aux_lob_piece" const uint64_t OB_ALL_COLUMN_AUX_LOB_PIECE_TID = 60004; // "__all_column_aux_lob_piece" const uint64_t OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TID = 60005; // "__all_ddl_operation_aux_lob_piece" @@ -2498,6 +2500,7 @@ const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_PIECE_TID = 60502; // "__ const uint64_t OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TID = 60505; // "__all_column_privilege_aux_lob_piece" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TID = 60506; // "__all_column_privilege_history_aux_lob_piece" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID = 60507; // "__all_tenant_snapshot_ls_replica_history_aux_lob_piece" +const uint64_t OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID = 60518; // "__all_mview_dep_aux_lob_piece" const uint64_t OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TID = 14999; // "__all_virtual_plan_cache_stat" const uint64_t OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TID = 14998; // "__all_virtual_session_event" const uint64_t OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TID = 14997; // "__all_virtual_session_wait" @@ -2989,6 +2992,7 @@ const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_TNAME = "__all_trusted_root_ce const char *const OB_ALL_COLUMN_PRIVILEGE_TNAME = "__all_column_privilege"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_TNAME = "__all_column_privilege_history"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME = "__all_tenant_snapshot_ls_replica_history"; +const char *const OB_ALL_MVIEW_DEP_TNAME = "__all_mview_dep"; const char *const OB_TENANT_VIRTUAL_ALL_TABLE_TNAME = "__tenant_virtual_all_table"; const char *const OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME = "__tenant_virtual_table_column"; const char *const OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME = "__tenant_virtual_table_index"; @@ -4873,6 +4877,7 @@ const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_META_TNAME = "__all_tr const char *const OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TNAME = "__all_column_privilege_aux_lob_meta"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TNAME = "__all_column_privilege_history_aux_lob_meta"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TNAME = "__all_tenant_snapshot_ls_replica_history_aux_lob_meta"; +const char *const OB_ALL_MVIEW_DEP_AUX_LOB_META_TNAME = "__all_mview_dep_aux_lob_meta"; const char *const OB_ALL_TABLE_AUX_LOB_PIECE_TNAME = "__all_table_aux_lob_piece"; const char *const OB_ALL_COLUMN_AUX_LOB_PIECE_TNAME = "__all_column_aux_lob_piece"; const char *const OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TNAME = "__all_ddl_operation_aux_lob_piece"; @@ -5166,6 +5171,7 @@ const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_PIECE_TNAME = "__all_t const char *const OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TNAME = "__all_column_privilege_aux_lob_piece"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_column_privilege_history_aux_lob_piece"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME = "__all_tenant_snapshot_ls_replica_history_aux_lob_piece"; +const char *const OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TNAME = "__all_mview_dep_aux_lob_piece"; const char *const OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TNAME = "__idx_11003_all_virtual_plan_cache_stat_i1"; const char *const OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TNAME = "__idx_11013_all_virtual_session_event_i1"; const char *const OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TNAME = "__idx_11014_all_virtual_session_wait_i1"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 4399e2c24..6a606666a 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -7128,7 +7128,26 @@ def_table_schema(**all_tenant_snapshot_ls_replica_history_def) # 515 : __all_user_proxy_role_info_history # 516 : __all_service # 517 : __all_storage_io_usage -# 518 : __all_mview_dep + +def_table_schema( + owner = 'yuya.yu', + table_name = '__all_mview_dep', + table_id = '518', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('mview_id', 'int'), + ('p_order', 'int') + ], + in_tenant_space = True, + normal_columns = [ + ('p_obj', 'int'), + ('p_type', 'int'), + ('qbcid', 'int'), + ('flags', 'int') + ] +) # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index ba1115c9e..0cfb820b6 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -350,6 +350,7 @@ # 506: __all_column_privilege_history # 506: __all_column_privilege # BASE_TABLE_NAME # 507: __all_tenant_snapshot_ls_replica_history +# 518: __all_mview_dep # 10001: __tenant_virtual_all_table # 10002: __tenant_virtual_table_column # 10003: __tenant_virtual_table_index diff --git a/src/share/ob_ddl_common.cpp b/src/share/ob_ddl_common.cpp index 996314164..3bd087a38 100644 --- a/src/share/ob_ddl_common.cpp +++ b/src/share/ob_ddl_common.cpp @@ -1104,6 +1104,7 @@ int ObDDLUtil::generate_build_mview_replica_sql( ObString database_name; ObString container_table_name; ObSqlString src_table_schema_version_hint; + ObSqlString rowkey_column_sql_string; if (OB_FAIL(sql::ObSQLUtils::generate_new_name_with_escape_character( allocator, database_schema->get_database_name_str(), database_name, is_oracle_mode))) { LOG_WARN("fail to generate new name with escape character", KR(ret), @@ -1139,29 +1140,31 @@ int ObDDLUtil::generate_build_mview_replica_sql( } } if (OB_SUCC(ret)) { - int64_t real_parallelism = std::max(1L, parallelism); + int64_t real_parallelism = std::max(2L, parallelism); real_parallelism = std::min(ObMacroDataSeq::MAX_PARALLEL_IDX + 1, real_parallelism); const ObString &select_sql_string = mview_table_schema->get_view_schema().get_view_definition_str(); if (is_oracle_mode) { - if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) opt_param('enable_newsort', 'false') use_px */ INTO \"%.*s\".\"%.*s\"" - " SELECT /*+ %.*s */ * from (%.*s) as of scn %ld;", - real_parallelism, execution_id, task_id, + if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ append monitor enable_parallel_dml parallel(%ld) opt_param('ddl_task_id', %ld) use_px */ INTO \"%.*s\".\"%.*s\"" + " SELECT /*+ %.*s */ * from (%.*s) as of scn %ld %.*s;", + real_parallelism, task_id, static_cast(database_name.length()), database_name.ptr(), static_cast(container_table_name.length()), container_table_name.ptr(), static_cast(src_table_schema_version_hint.length()), src_table_schema_version_hint.ptr(), static_cast(select_sql_string.length()), select_sql_string.ptr(), - snapshot_version))) { + snapshot_version, + static_cast(rowkey_column_sql_string.length()), rowkey_column_sql_string.ptr()))) { LOG_WARN("fail to assign sql string", KR(ret)); } } else { - if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) opt_param('enable_newsort', 'false') use_px */ INTO `%.*s`.`%.*s`" - " SELECT /*+ %.*s */ * from (%.*s) as of snapshot %ld;", - real_parallelism, execution_id, task_id, + if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ append monitor enable_parallel_dml parallel(%ld) opt_param('ddl_task_id', %ld) use_px */ INTO `%.*s`.`%.*s`" + " SELECT /*+ %.*s */ * from (%.*s) as of snapshot %ld %.*s;", + real_parallelism, task_id, static_cast(database_name.length()), database_name.ptr(), static_cast(container_table_name.length()), container_table_name.ptr(), static_cast(src_table_schema_version_hint.length()), src_table_schema_version_hint.ptr(), static_cast(select_sql_string.length()), select_sql_string.ptr(), - snapshot_version))) { + snapshot_version, + static_cast(rowkey_column_sql_string.length()), rowkey_column_sql_string.ptr()))) { LOG_WARN("fail to assign sql string", KR(ret)); } } @@ -1172,6 +1175,36 @@ int ObDDLUtil::generate_build_mview_replica_sql( return ret; } +int ObDDLUtil::generate_order_by_str_for_mview(const ObTableSchema &container_table_schema, + ObSqlString &rowkey_column_sql_string) +{ + int ret = OB_SUCCESS; + rowkey_column_sql_string.reset(); + const ObRowkeyInfo &rowkey_info = container_table_schema.get_rowkey_info(); + if (container_table_schema.is_heap_table()) { + /* do nothing */ + } else if (OB_UNLIKELY(rowkey_info.get_size() < 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpcted rowkey size", K(ret), K(rowkey_info.get_size())); + } else if (OB_FAIL(rowkey_column_sql_string.append(" order by "))) { + LOG_WARN("append failed", KR(ret)); + } else { + uint64_t column_id = OB_INVALID_ID; + int64_t sel_pos = 0; + for (int col_idx = 0; OB_SUCC(ret) && col_idx < rowkey_info.get_size(); ++col_idx) { + if (OB_FAIL(rowkey_info.get_column_id(col_idx, column_id))) { + LOG_WARN("Failed to get column id", K(ret)); + } else if (OB_UNLIKELY(1 > (sel_pos = column_id - OB_APP_MIN_COLUMN_ID + 1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpcted select position", K(ret), K(sel_pos), K(column_id)); + } else if (OB_FAIL(rowkey_column_sql_string.append_fmt("%s %ld", 0 != col_idx ? ",": "", sel_pos))) { + LOG_WARN("append fmt failed", K(ret)); + } + } + } + return ret; +} + int ObDDLUtil::get_tablet_leader_addr( share::ObLocationService *location_service, const uint64_t tenant_id, @@ -1490,11 +1523,35 @@ int ObDDLUtil::get_data_information( uint64_t &data_format_version, int64_t &snapshot_version, share::ObDDLTaskStatus &task_status) +{ + uint64_t target_object_id = 0; + int64_t schema_version = 0; + + return get_data_information( + tenant_id, + task_id, + data_format_version, + snapshot_version, + task_status, + target_object_id, + schema_version); +} + +int ObDDLUtil::get_data_information( + const uint64_t tenant_id, + const uint64_t task_id, + uint64_t &data_format_version, + int64_t &snapshot_version, + share::ObDDLTaskStatus &task_status, + uint64_t &target_object_id, + int64_t &schema_version) { int ret = OB_SUCCESS; data_format_version = 0; snapshot_version = 0; task_status = share::ObDDLTaskStatus::PREPARE; + target_object_id = 0; + data_format_version = 0; if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || task_id <= 0 || nullptr == GCTX.sql_proxy_)) { ret = OB_INVALID_ARGUMENT; @@ -1505,7 +1562,7 @@ int ObDDLUtil::get_data_information( SMART_VAR(ObMySQLProxy::MySQLResult, res) { ObSqlString query_string; sqlclient::ObMySQLResult *result = NULL; - if (OB_FAIL(query_string.assign_fmt(" SELECT snapshot_version, ddl_type, UNHEX(message) as message_unhex, status FROM %s WHERE task_id = %lu", + if (OB_FAIL(query_string.assign_fmt(" SELECT snapshot_version, ddl_type, UNHEX(message) as message_unhex, status, schema_version, target_object_id FROM %s WHERE task_id = %lu", OB_ALL_DDL_TASK_STATUS_TNAME, task_id))) { LOG_WARN("assign sql string failed", K(ret)); } else if (OB_FAIL(GCTX.sql_proxy_->read(res, tenant_id, query_string.ptr()))) { @@ -1524,6 +1581,9 @@ int ObDDLUtil::get_data_information( EXTRACT_INT_FIELD_MYSQL(*result, "ddl_type", ddl_type, ObDDLType); EXTRACT_VARCHAR_FIELD_MYSQL(*result, "message_unhex", task_message); EXTRACT_INT_FIELD_MYSQL(*result, "status", cur_task_status, int); + EXTRACT_INT_FIELD_MYSQL(*result, "target_object_id", target_object_id, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "schema_version", schema_version, int64_t); + task_status = static_cast(cur_task_status); if (ObDDLType::DDL_CREATE_INDEX == ddl_type) { SMART_VAR(rootserver::ObIndexBuildTask, task) { @@ -1956,6 +2016,11 @@ int ObDDLUtil::batch_check_tablet_checksum( return ret; } +bool ObDDLUtil::use_idempotent_mode(const int64_t data_format_version) +{ + return data_format_version >= DATA_VERSION_4_3_0_1; +} + /****************** ObCheckTabletDataComplementOp *************/ int ObCheckTabletDataComplementOp::check_task_inner_sql_session_status( @@ -2786,4 +2851,4 @@ void ObDDLEventInfo::copy_event(const ObDDLEventInfo &other) parent_trace_id_ = other.parent_trace_id_; trace_id_ = other.trace_id_; event_ts_ = other.event_ts_; -} \ No newline at end of file +} diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index 45ea715b2..68e612109 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -476,6 +476,9 @@ public: const bool is_oracle_mode, ObSqlString &sql_string); + static int generate_order_by_str_for_mview(const schema::ObTableSchema &container_table_schema, + ObSqlString &rowkey_column_sql_string); + static int ddl_get_tablet( storage::ObLSHandle &ls_handle, const ObTabletID &tablet_id, @@ -527,6 +530,15 @@ public: int64_t &snapshot_version, share::ObDDLTaskStatus &task_status); + static int get_data_information( + const uint64_t tenant_id, + const uint64_t task_id, + uint64_t &data_format_version, + int64_t &snapshot_version, + share::ObDDLTaskStatus &task_status, + uint64_t &target_object_id, + int64_t &schema_version); + static int replace_user_tenant_id( const ObDDLType &ddl_type, const uint64_t tenant_id, @@ -638,6 +650,7 @@ public: } return res; } + static bool use_idempotent_mode(const int64_t data_format_version); private: static int batch_check_tablet_checksum( @@ -771,7 +784,6 @@ private: const int64_t execution_id, const ObIArray &tablet_ids, bool &tablet_checksum_status); - }; typedef common::ObCurTraceId::TraceId DDLTraceId; diff --git a/src/share/ob_ddl_task_executor.h b/src/share/ob_ddl_task_executor.h index 96ff99470..95cf9a17c 100644 --- a/src/share/ob_ddl_task_executor.h +++ b/src/share/ob_ddl_task_executor.h @@ -63,7 +63,7 @@ public: } static bool in_ddl_retry_black_list(const int ret_code) { - return common::OB_SERVER_OUTOF_DISK_SPACE == ret_code || common::OB_DISK_ERROR == ret_code; + return common::OB_SERVER_OUTOF_DISK_SPACE == ret_code || common::OB_DISK_ERROR == ret_code || OB_NOT_SUPPORTED == ret_code; } static bool is_ddl_force_no_more_process(const int ret_code) { diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 66b5087a7..ec2f495c1 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -6473,8 +6473,8 @@ int ObPartitionUtils::check_param_valid_( index_exist = true; } } // end for simple_index_infos - if (OB_SUCC(ret) && !finded && table_schema.has_mlog_table()) { - finded = (related_tid == table_schema.get_mlog_tid()); + if (OB_SUCC(ret) && !finded && data_schema->has_mlog_table()) { + finded = (related_tid == data_schema->get_mlog_tid()); } if (OB_SUCC(ret) && !finded && related_tid != data_table_id) { ret = OB_TABLE_NOT_EXIST; diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index 8885c4fd4..ed926935c 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -746,6 +746,44 @@ int ObSchemaUtils::alter_rowkey_column_group(share::schema::ObTableSchema &table return ret; } +int ObSchemaUtils::check_sys_table_exist_by_sql( + common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const ObObjectID &table_id, + bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || OB_INVALID_ID == table_id + || !is_sys_table(table_id) + || is_core_table(table_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tenant_id), K(table_id)); + } else { + SMART_VAR(ObISQLClient::ReadResult, result) { + ObSqlString sql; + common::sqlclient::ObMySQLResult *res = NULL; + // in __all_table, tenant_id is primary key and it's value is 0 + if (OB_FAIL(sql.append_fmt( + "SELECT count(*) = 1 AS exist FROM %s WHERE tenant_id = 0 and table_id = %lu", + OB_ALL_TABLE_TNAME, table_id))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(result, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_ISNULL(res = result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql)); + } else if (OB_FAIL(res->next())) { + LOG_WARN("next failed", KR(ret), K(sql)); + } else if (OB_FAIL(res->get_bool("exist", exist))) { + LOG_WARN("get bool value failed", KR(ret), K(sql)); + } + } + } + return ret; +} + /* check all column group add add not exi*/ int ObSchemaUtils::alter_default_column_group(share::schema::ObTableSchema &new_table_schema) { diff --git a/src/share/schema/ob_schema_utils.h b/src/share/schema/ob_schema_utils.h index 3cd74ca1c..d4e7b62de 100644 --- a/src/share/schema/ob_schema_utils.h +++ b/src/share/schema/ob_schema_utils.h @@ -188,6 +188,21 @@ public: const uint64_t tenant_id, const int64_t schema_version, const bool skip_consensus); + + // Use to check if the sys table (exclude core table) does exist + // by querying __all_table when the table is not accessible. + // + // @param[in] sql_client: ObISQLClient + // @param[in] tenant_id: target tenant_id + // @param[in] table_id: sys table_id (exclude core table) + // @param[out] exist: whether the table really exists + // @return: OB_SUCCESS if success + static int check_sys_table_exist_by_sql( + common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const ObObjectID &table_id, + bool &exist); + private: static int get_tenant_variable(schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 635e60ea4..591aa36b2 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -378,7 +378,10 @@ public: "rowid_mode", rowid_mode_, "view_column_filled_flag", view_column_filled_flag_, "mv_container_table_flag", mv_container_table_flag_, - "mv_available_flag", mv_available_flag_); + "mv_available_flag", mv_available_flag_, + "table_referenced_by_mv_flag", table_referenced_by_mv_flag_, + "mv_enable_query_rewrite_flag", mv_enable_query_rewrite_flag_, + "mv_on_query_computation_flag", mv_on_query_computation_flag_); union { int32_t mode_; struct { @@ -392,6 +395,9 @@ public: uint32_t view_column_filled_flag_ : TM_VIEW_COLUMN_FILLED_BITS; uint32_t mv_container_table_flag_ : TM_MV_CONTAINER_TABLE_BITS; uint32_t mv_available_flag_ : TM_MV_AVAILABLE_BITS; + uint32_t table_referenced_by_mv_flag_ : TM_TABLE_REFERENCED_BY_MV_BITS; + uint32_t mv_enable_query_rewrite_flag_ : TM_MV_ENABLE_QUERY_REWRITE_BITS; + uint32_t mv_on_query_computation_flag_ : TM_MV_ON_QUERY_COMPUTATION_BITS; uint32_t reserved_ :TM_RESERVED; }; }; @@ -705,6 +711,18 @@ public: { return IS_MV_AVAILABLE == (enum ObMVAvailableFlag)table_mode_.mv_available_flag_; } inline void set_mv_available(const ObMVAvailableFlag flag) { table_mode_.mv_available_flag_ = flag; } + inline bool table_referenced_by_mv() const + { return IS_REFERENCED_BY_MV == (enum ObTableReferencedByMVFlag)table_mode_.table_referenced_by_mv_flag_; } + inline void set_table_referenced_by_mv(const ObTableReferencedByMVFlag flag) + { table_mode_.table_referenced_by_mv_flag_ = flag; } + inline bool mv_enable_query_rewrite() const + { return IS_MV_ENABLE_QUERY_REWRITE == (enum ObMVEnableQueryRewriteFlag)table_mode_.mv_enable_query_rewrite_flag_; } + inline void set_mv_enable_query_rewrite(const ObMVEnableQueryRewriteFlag flag) + { table_mode_.mv_enable_query_rewrite_flag_ = flag; } + inline bool mv_on_query_computation() const + { return IS_MV_ON_QUERY_COMPUTATION == (enum ObMVOnQueryComputationFlag)table_mode_.mv_on_query_computation_flag_; } + inline void set_mv_on_query_computation(const ObMVOnQueryComputationFlag flag) + { table_mode_.mv_on_query_computation_flag_ = flag; } inline void set_session_id(const uint64_t id) { session_id_ = id; } inline uint64_t get_session_id() const { return session_id_; } diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index a0815017f..f63ceab74 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -2727,6 +2727,77 @@ int ObTableSqlService::update_mview_status( return ret; } +int ObTableSqlService::update_mview_reference_table_status( + const ObTableSchema &table_schema, + common::ObISQLClient &sql_client) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + const uint64_t tenant_id = table_schema.get_tenant_id(); + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + share::schema::ObTableMode table_mode_struct = table_schema.get_table_mode_struct(); + uint64_t table_id = table_schema.get_table_id(); + int64_t new_schema_version = table_schema.get_schema_version(); + ObDMLSqlSplicer dml; + if (OB_FAIL(dml.add_pk_column("tenant_id", ObSchemaUtils::get_extract_tenant_id( + exec_tenant_id, tenant_id))) + || OB_FAIL(dml.add_pk_column("table_id", ObSchemaUtils::get_extract_schema_id( + exec_tenant_id, table_id))) + || OB_FAIL(dml.add_column("schema_version", new_schema_version)) + || OB_FAIL(dml.add_column("table_mode", table_mode_struct.mode_))) { + LOG_WARN("failed to add column", KR(ret), K(exec_tenant_id), K(tenant_id)); + } else { + int64_t affected_rows = 0; + const char *table_name = NULL; + if (OB_FAIL(ObSchemaUtils::get_all_table_name(exec_tenant_id, table_name))) { + LOG_WARN("fail to get all table name", KR(ret), K(exec_tenant_id)); + } else if (OB_FAIL(exec_update(sql_client, + tenant_id, table_id, table_name, dml, affected_rows))) { + LOG_WARN("failed to exec update", KR(ret), + K(tenant_id), K(table_id), K(table_name)); + } else if (!is_single_row(affected_rows)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret), K(affected_rows)); + } + } + + if (OB_SUCC(ret)) { + ObRefreshSchemaStatus schema_status; + schema_status.tenant_id_ = tenant_id; + const bool update_object_status_ignore_version = false; + SMART_VAR(ObTableSchema, new_schema) { + if (OB_FAIL(schema_service_.get_table_schema_from_inner_table( + schema_status, table_id, sql_client, new_schema))) { + LOG_WARN("failed to get table schema for inner table", KR(ret), K(table_id)); + } else { + const bool only_history = true; + new_schema.set_table_referenced_by_mv(ObTableMode::get_table_referenced_by_mv_flag(table_mode_struct.mode_)); + new_schema.set_schema_version(new_schema_version); + new_schema.set_in_offline_ddl_white_list(table_schema.get_in_offline_ddl_white_list()); + if (OB_FAIL(add_table(sql_client, + new_schema, + update_object_status_ignore_version, + only_history))) { + LOG_WARN("failed to add_table", KR(ret), K(new_schema), K(only_history)); + } + } + } + } + + if (OB_SUCC(ret)) { + ObSchemaOperation opt; + opt.tenant_id_ = tenant_id; + opt.database_id_ = 0; + opt.tablegroup_id_ = 0; + opt.table_id_ = table_id; + opt.op_type_ = OB_DDL_MODIFY_MVIEW_REFERENCE_TABLE_STATUS; + opt.schema_version_ = new_schema_version; + if (OB_FAIL(log_operation_wrapper(opt, sql_client))) { + LOG_WARN("log operation failed", K(ret), K(opt)); + } + } + return ret; +} int ObTableSqlService::add_transition_point_val(ObDMLSqlSplicer &dml, const ObTableSchema &table) { diff --git a/src/share/schema/ob_table_sql_service.h b/src/share/schema/ob_table_sql_service.h index 49906f0c4..628957615 100644 --- a/src/share/schema/ob_table_sql_service.h +++ b/src/share/schema/ob_table_sql_service.h @@ -136,6 +136,8 @@ public: virtual int update_mview_status(const ObTableSchema &mview_table_schema, common::ObISQLClient &sql_client); + virtual int update_mview_reference_table_status(const ObTableSchema &table_schema, + common::ObISQLClient &sql_client); // TODO: merge these two API int sync_aux_schema_version_for_history(common::ObISQLClient &sql_client, const ObTableSchema &index_schema1, diff --git a/src/share/system_variable/ob_system_variable_init.cpp b/src/share/system_variable/ob_system_variable_init.cpp index 01d93ca9b..a26abeb05 100644 --- a/src/share/system_variable/ob_system_variable_init.cpp +++ b/src/share/system_variable/ob_system_variable_init.cpp @@ -3709,7 +3709,7 @@ static struct VarsInit{ }(); [&] (){ - ObSysVars[263].default_value_ = "1" ; + ObSysVars[263].default_value_ = "0" ; ObSysVars[263].info_ = "Whether to enable the materialized view rewriting" ; ObSysVars[263].name_ = "query_rewrite_enabled" ; ObSysVars[263].data_type_ = ObIntType ; @@ -3718,7 +3718,7 @@ static struct VarsInit{ ObSysVars[263].id_ = SYS_VAR_QUERY_REWRITE_ENABLED ; cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_QUERY_REWRITE_ENABLED)) ; ObSysVarsIdToArrayIdx[SYS_VAR_QUERY_REWRITE_ENABLED] = 263 ; - ObSysVars[263].base_value_ = "1" ; + ObSysVars[263].base_value_ = "0" ; ObSysVars[263].alias_ = "OB_SV_QUERY_REWRITE_ENABLED" ; }(); diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index 79628bf63..de30093a2 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -3760,8 +3760,8 @@ "query_rewrite_enabled": { "id": 10179, "name": "query_rewrite_enabled", - "default_value": "1", - "base_value": "1", + "default_value": "0", + "base_value": "0", "data_type": "enum", "info": "Whether to enable the materialized view rewriting", "flags": "GLOBAL | SESSION | NEED_SERIALIZE | INFLUENCE_PLAN", diff --git a/src/share/table/ob_table_load_define.cpp b/src/share/table/ob_table_load_define.cpp index 9e23888dd..c3a5363e3 100644 --- a/src/share/table/ob_table_load_define.cpp +++ b/src/share/table/ob_table_load_define.cpp @@ -19,36 +19,36 @@ namespace oceanbase namespace table { -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadConfig, - parallel_, - batch_size_, - max_error_row_count_, - dup_action_, - is_need_sort_); +OB_SERIALIZE_MEMBER(ObTableLoadConfig, + parallel_, + batch_size_, + max_error_row_count_, + dup_action_, + is_need_sort_); -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadSegmentID, - id_); +OB_SERIALIZE_MEMBER(ObTableLoadSegmentID, + id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadTransId, - segment_id_, - trans_gid_); +OB_SERIALIZE_MEMBER(ObTableLoadTransId, + segment_id_, + trans_gid_); -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadPartitionId, - partition_id_, - tablet_id_); +OB_SERIALIZE_MEMBER(ObTableLoadPartitionId, + partition_id_, + tablet_id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadLSIdAndPartitionId, - ls_id_, - part_tablet_id_); +OB_SERIALIZE_MEMBER(ObTableLoadLSIdAndPartitionId, + ls_id_, + part_tablet_id_); -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadResultInfo, - rows_affected_, - records_, - deleted_, - skipped_, - warnings_); +OB_SERIALIZE_MEMBER(ObTableLoadResultInfo, + rows_affected_, + records_, + deleted_, + skipped_, + warnings_); -OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadSequenceNo, sequence_no_); +OB_SERIALIZE_MEMBER(ObTableLoadSequenceNo, sequence_no_); } // namespace table } // namespace oceanbase diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index fe90feef6..ba38e479e 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -33,6 +33,7 @@ ob_set_subtarget(ob_sql common_mixed resolver/mv/ob_mv_checker.cpp resolver/mv/ob_mv_provider.cpp resolver/mv/ob_mv_printer.cpp + resolver/mv/ob_mv_dep_utils.cpp ) ob_set_subtarget(ob_sql das @@ -719,6 +720,7 @@ ob_set_subtarget(ob_sql engine_expr engine/expr/ob_expr_extract_cert_expired_time.cpp engine/expr/ob_expr_transaction_id.cpp engine/expr/ob_expr_inner_row_cmp_val.cpp + engine/expr/ob_expr_last_refresh_scn.cpp engine/expr/ob_expr_json_utils.cpp ) @@ -1282,6 +1284,7 @@ ob_set_subtarget(ob_sql rewrite rewrite/ob_transform_expr_pullup.cpp rewrite/ob_transformer_impl.cpp rewrite/ob_transform_dblink.cpp + rewrite/ob_transform_mv_rewrite.cpp rewrite/ob_union_find.cpp rewrite/ob_transform_decorrelate.cpp ) diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index f940c9533..500972c48 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -1508,14 +1508,20 @@ int ObDmlCgService::check_upd_need_all_columns(ObLogDelUpd &op, need_all_columns = false; int64_t binlog_row_image = ObBinlogRowImage::FULL; ObSQLSessionInfo *session = cg_.opt_ctx_->get_session_info(); - if (OB_ISNULL(session)) { + if (OB_ISNULL(session) || OB_ISNULL(schema_guard) || OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null ptr", K(ret)); + LOG_WARN("unexpected null ptr", K(ret), KP(session), KP(schema_guard), KP(table_schema)); } else if (OB_FAIL(session->get_binlog_row_image(binlog_row_image))) { LOG_WARN("fail to get binlog image", K(ret)); } else if (binlog_row_image == ObBinlogRowImage::FULL || op.has_instead_of_trigger()) { // full mode need_all_columns = true; + } else if (table_schema->has_mlog_table()) { + need_all_columns = true; + LOG_TRACE("update table with materialized view log, need all columns", K(table_schema->has_mlog_table())); + } else if (table_schema->is_mlog_table()) { + need_all_columns = true; + LOG_TRACE("update materialized view log, need all columns", K(table_schema->is_mlog_table())); } else if (!is_primary_index) { // index_table if update PK, also need record all_columns if (OB_FAIL(check_has_upd_rowkey(op, table_schema, index_dml_info, is_update_pk))) { @@ -1658,14 +1664,20 @@ int ObDmlCgService::check_del_need_all_columns(ObLogDelUpd &op, bool has_not_null_uk = false; int64_t binlog_row_image = ObBinlogRowImage::FULL; ObSQLSessionInfo *session = cg_.opt_ctx_->get_session_info(); - if (OB_ISNULL(session)) { + if (OB_ISNULL(session) || OB_ISNULL(schema_guard) || OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null ptr", K(ret)); + LOG_WARN("unexpected null ptr", K(ret), KP(session), KP(schema_guard), KP(table_schema)); } else if (OB_FAIL(session->get_binlog_row_image(binlog_row_image))) { LOG_WARN("fail to get binlog image", K(ret)); } else if (binlog_row_image == ObBinlogRowImage::FULL || op.has_instead_of_trigger()) { // full mode need_all_columns = true; + } else if (table_schema->has_mlog_table()) { + need_all_columns = true; + LOG_TRACE("delete from table with materialized view log, need all columns", K(table_schema->has_mlog_table())); + } else if (table_schema->is_mlog_table()) { + need_all_columns = true; + LOG_TRACE("delete from materialized view log, need all columns", K(table_schema->is_mlog_table())); } else if (table_schema->is_heap_table()) { if (OB_FAIL(table_schema->has_not_null_unique_key(*schema_guard, has_not_null_uk))) { LOG_WARN("fail to check whether has not null unique key", K(ret), K(table_schema->get_table_name_str())); diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 9279298de..335955af8 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -843,6 +843,11 @@ int ObStaticEngineCG::generate_spec_basic(ObLogicalOperator &op, if (OB_SUCC(ret) && need_check_output_datum) { OZ(add_output_datum_check_flag(spec)); } + if (OB_SUCC(ret)) { + if (OB_FAIL(extract_all_mview_ids(cur_op_exprs_))) { + LOG_WARN("fail to extract all mview ids", K(ret)); + } + } if (OB_SUCC(ret)) { CK (OB_NOT_NULL(op.get_plan()) && OB_NOT_NULL(op.get_plan()->get_stmt()) @@ -862,6 +867,37 @@ int ObStaticEngineCG::generate_spec_basic(ObLogicalOperator &op, return ret; } +int ObStaticEngineCG::extract_all_mview_ids(const ObIArray &exprs) +{ + int ret = OB_SUCCESS; + for (int i = 0; OB_SUCC(ret) && i < exprs.count(); ++i) { + if (OB_FAIL(extract_all_mview_ids(exprs.at(i)))) { + LOG_WARN("extract all mview ids failed", K(ret), K(i), KPC(exprs.at(i))); + } + } + return ret; +} + +int ObStaticEngineCG::extract_all_mview_ids(const ObRawExpr *expr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret), K(expr)); + } else if (ObItemType::T_FUN_SYS_LAST_REFRESH_SCN == expr->get_expr_type()) { + if (OB_FAIL(add_var_to_array_no_dup(mview_ids_, static_cast(expr)->get_mview_id()))) { + LOG_WARN("failed to add var to array no dup", K(ret), K(mview_ids_.count())); + } + } else { + for (int i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { + if (OB_FAIL(SMART_CALL(extract_all_mview_ids(expr->get_param_expr(i))))) { + LOG_WARN("extract all mview ids failed", K(ret), KPC(expr->get_param_expr(i))); + } + } + } + return ret; +} + int ObStaticEngineCG::generate_calc_exprs( const ObIArray &dep_exprs, const ObIArray &cur_exprs, @@ -7883,6 +7919,8 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical LOG_WARN("set expected worker map", K(ret)); } else if (OB_FAIL(phy_plan.set_minimal_worker_map(log_plan.get_optimizer_context().get_minimal_worker_map()))) { LOG_WARN("set minimal worker map", K(ret)); + } else if (OB_FAIL(phy_plan.set_mview_ids(mview_ids_))) { + LOG_WARN("failed to set set mview_ids", K(ret), K(mview_ids_.count())); } else { if (log_plan.get_optimizer_context().is_online_ddl()) { if (log_plan.get_stmt()->get_table_items().count() > 0) { @@ -7900,6 +7938,17 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical } } } + if (log_plan.get_optimizer_context().get_session_info()->get_ddl_info().is_mview_complete_refresh()) { + if (log_plan.get_stmt()->get_table_items().count() > 0) { + const TableItem *insert_table_item = log_plan.get_stmt()->get_table_item(0); + if (nullptr != insert_table_item) { + int64_t ddl_task_id = 0; + const ObOptParamHint *opt_params = &log_plan.get_stmt()->get_query_ctx()->get_global_hint().opt_params_; + OZ(opt_params->get_integer_opt_param(ObOptParamHint::DDL_TASK_ID, ddl_task_id)); + log_plan.get_optimizer_context().get_exec_ctx()->get_table_direct_insert_ctx().set_ddl_task_id(ddl_task_id); + } + } + } } ObParamOption param_opt = log_plan.get_optimizer_context().get_global_hint().param_option_; bool is_exact_mode = my_session->get_enable_exact_mode(); diff --git a/src/sql/code_generator/ob_static_engine_cg.h b/src/sql/code_generator/ob_static_engine_cg.h index 6993496c7..2c7635cf5 100644 --- a/src/sql/code_generator/ob_static_engine_cg.h +++ b/src/sql/code_generator/ob_static_engine_cg.h @@ -558,6 +558,8 @@ private: int generate_sort_exprs(const bool is_store_sortkey_separately, ObLogSort &op, ObSortVecSpec &spec, ObIArray &sk_keys); + int extract_all_mview_ids(const ObIArray &exprs); + int extract_all_mview_ids(const ObRawExpr *expr); private: struct BatchExecParamCache { BatchExecParamCache(ObExecParamRawExpr* expr, ObOpSpec* spec, bool is_left) @@ -594,6 +596,7 @@ private: ObTscCgService tsc_cg_service_; uint64_t cur_cluster_version_; common::ObSEArray batch_exec_param_caches_; + common::ObSEArray mview_ids_; }; diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index 2da092119..1318c64da 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -482,18 +482,17 @@ int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id, } else if (OB_FAIL(auto_inc.get_autoinc_seq(tenant_id, tablet_id, autoinc_seq))) { LOG_WARN("get_autoinc_seq fail", K(ret), K(tenant_id), K(tablet_id)); } else { - // sequence_col is the first primary key + // mlog_row = | base_table_rowkey_cols | partition key cols | sequence_col | ... | dmltype_col | old_new_col | int sequence_col = 0; int dmltype_col = row.count_ - 2; int old_new_col = row.count_ - 1; const ObTableDMLParam::ObColDescArray &col_descs = dml_param.table_param_->get_col_descs(); - bool is_heap_base_table = (OB_MLOG_ROWID_COLUMN_ID == col_descs.at(row.count_ - 1).col_id_); - // if the base table is heap table, then the last column is mlog_rowid, - // therefore, row = | sequence_col | partition key cols | ... | dmltype_col | old_new_col | rowid_col | - // otherwise, row = | sequence_col | partition key cols | ... | dmltype_col | old_new_col | - if (is_heap_base_table) { - dmltype_col = dmltype_col - 1; - old_new_col = old_new_col - 1; + bool found_seq_col = false; + for (int64_t i = 0; !found_seq_col && (i < row.count_); ++i) { + if (OB_MLOG_SEQ_NO_COLUMN_ID == col_descs.at(i).col_id_) { + sequence_col = i; // sequence_no is the last rowkey + found_seq_col = true; + } } row.cells_[sequence_col].set_int(ObObjType::ObIntType, static_cast(autoinc_seq)); diff --git a/src/sql/engine/cmd/ob_ddl_executor_util.cpp b/src/sql/engine/cmd/ob_ddl_executor_util.cpp index ad11e6f35..63032f5ba 100644 --- a/src/sql/engine/cmd/ob_ddl_executor_util.cpp +++ b/src/sql/engine/cmd/ob_ddl_executor_util.cpp @@ -48,6 +48,7 @@ int ObDDLExecutorUtil::handle_session_exception(ObSQLSessionInfo &session) int ObDDLExecutorUtil::wait_ddl_finish( const uint64_t tenant_id, const int64_t task_id, + const bool ddl_need_retry_at_executor, ObSQLSessionInfo *session, obrpc::ObCommonRpcProxy *common_rpc_proxy, const bool is_support_cancel) @@ -79,7 +80,12 @@ int ObDDLExecutorUtil::wait_ddl_finish( tenant_id, task_id, -1 /* target_object_id */, unused_addr, false /* is_ddl_retry_task */, *GCTX.sql_proxy_, error_message, unused_user_msg_len)) { ret = error_message.ret_code_; if (OB_SUCCESS != ret) { - FORWARD_USER_ERROR(ret, error_message.user_message_); + if (ddl_need_retry_at_executor) { + ret = ObIDDLTask::in_ddl_retry_white_list(ret) ? OB_EAGAIN : ret; + LOG_WARN("is ddl need retry at user", K(ret)); + } else { + FORWARD_USER_ERROR(ret, error_message.user_message_); + } } break; } else { diff --git a/src/sql/engine/cmd/ob_ddl_executor_util.h b/src/sql/engine/cmd/ob_ddl_executor_util.h index dda17da1f..9d7c8491d 100644 --- a/src/sql/engine/cmd/ob_ddl_executor_util.h +++ b/src/sql/engine/cmd/ob_ddl_executor_util.h @@ -55,6 +55,7 @@ public: static int wait_ddl_finish( const uint64_t tenant_id, const int64_t task_id, + const bool ddl_need_retry_at_executor, ObSQLSessionInfo *session, obrpc::ObCommonRpcProxy *common_rpc_proxy, const bool is_support_cancel = true); diff --git a/src/sql/engine/cmd/ob_index_executor.cpp b/src/sql/engine/cmd/ob_index_executor.cpp index 2aed455a7..a95dc40ac 100644 --- a/src/sql/engine/cmd/ob_index_executor.cpp +++ b/src/sql/engine/cmd/ob_index_executor.cpp @@ -99,7 +99,7 @@ int ObCreateIndexExecutor::execute(ObExecContext &ctx, ObCreateIndexStmt &stmt) ret = OB_ERR_ADD_INDEX; LOG_WARN("index table id is invalid", KR(ret)); } - } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(create_index_arg.tenant_id_, res.task_id_, my_session, common_rpc_proxy))) { + } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(create_index_arg.tenant_id_, res.task_id_, false/*do not need retry at executor*/, my_session, common_rpc_proxy))) { LOG_WARN("failed to wait ddl finish", K(ret)); } } diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index d7d498902..dc36c97e0 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -31,6 +31,7 @@ namespace sql using namespace common; using namespace observer; using namespace share; +using namespace storage; using namespace table; using namespace omt; @@ -2042,6 +2043,8 @@ int ObLoadDataDirectImpl::init_execute_context() load_param.sql_mode_ = execute_param_.sql_mode_; load_param.px_mode_ = false; load_param.online_opt_stat_gather_ = execute_param_.online_opt_stat_gather_; + load_param.method_ = ObDirectLoadMethod::FULL; + load_param.insert_mode_ = ObDirectLoadInsertMode::NORMAL; if (OB_FAIL(direct_loader_.init(load_param, execute_param_.store_column_idxs_, &execute_ctx_.exec_ctx_))) { LOG_WARN("fail to init direct loader", KR(ret)); diff --git a/src/sql/engine/cmd/ob_mlog_executor.cpp b/src/sql/engine/cmd/ob_mlog_executor.cpp index 66815ee90..ae848fe9e 100644 --- a/src/sql/engine/cmd/ob_mlog_executor.cpp +++ b/src/sql/engine/cmd/ob_mlog_executor.cpp @@ -75,7 +75,7 @@ int ObCreateMLogExecutor::execute(ObExecContext &ctx, ObCreateMLogStmt &stmt) ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected schema version", KR(ret), K(create_mlog_res)); } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(create_mlog_arg.tenant_id_, - create_mlog_res.task_id_, my_session, common_rpc_proxy))) { + create_mlog_res.task_id_, false/*do not need retry at executor*/, my_session, common_rpc_proxy))) { LOG_WARN("failed to wait ddl finish", KR(ret)); } } diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp index 64b999d79..bbfa91877 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp @@ -55,15 +55,12 @@ int ObTableDirectInsertCtx::init(ObExecContext *exec_ctx, LOG_WARN("fail to new ObTableLoadInstance", KR(ret)); } else { load_exec_ctx_->exec_ctx_ = exec_ctx; - uint64_t sql_mode = 0; ObSEArray store_column_idxs; omt::ObTenant *tenant = nullptr; if (OB_FAIL(GCTX.omt_->get_tenant(MTL_ID(), tenant))) { LOG_WARN("fail to get tenant handle", KR(ret), K(MTL_ID())); } else if (OB_FAIL(init_store_column_idxs(MTL_ID(), table_id, store_column_idxs))) { LOG_WARN("failed to init store column idxs", KR(ret)); - } else if (OB_FAIL(exec_ctx->get_my_session()->get_sys_variable(SYS_VAR_SQL_MODE, sql_mode))) { - LOG_WARN("fail to get sys variable", KR(ret)); } else { ObTableLoadParam param; param.column_count_ = store_column_idxs.count(); @@ -77,8 +74,11 @@ int ObTableDirectInsertCtx::init(ObExecContext *exec_ctx, param.need_sort_ = true; param.max_error_row_count_ = 0; param.dup_action_ = sql::ObLoadDupActionType::LOAD_STOP_ON_DUP; - param.sql_mode_ = sql_mode; param.online_opt_stat_gather_ = is_online_gather_statistics_; + param.method_ = ObDirectLoadMethod::FULL; + param.insert_mode_ = (exec_ctx->get_my_session()->get_ddl_info().is_mview_complete_refresh() + ? ObDirectLoadInsertMode::OVERWRITE + : ObDirectLoadInsertMode::NORMAL); if (OB_FAIL(table_load_instance_->init(param, store_column_idxs, load_exec_ctx_))) { LOG_WARN("failed to init direct loader", KR(ret)); } else { diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.h b/src/sql/engine/cmd/ob_table_direct_insert_ctx.h index ab3433108..4ee8894fc 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.h +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.h @@ -34,7 +34,8 @@ public: table_load_instance_(nullptr), is_inited_(false), is_direct_(false), - is_online_gather_statistics_(false) {} + is_online_gather_statistics_(false), + ddl_task_id_(0) {} ~ObTableDirectInsertCtx(); TO_STRING_KV(K_(is_inited)); public: @@ -53,6 +54,14 @@ public: is_online_gather_statistics_ = is_online_gather_statistics; } + void set_ddl_task_id(const int64_t ddl_task_id) { + ddl_task_id_ = ddl_task_id; + } + + int64_t get_ddl_task_id() const { + return ddl_task_id_; + } + private: int init_store_column_idxs(const uint64_t tenant_id, const uint64_t table_id, common::ObIArray &store_column_idxs); @@ -62,6 +71,7 @@ private: bool is_inited_; bool is_direct_; //indict whether the plan is direct load plan including insert into append and load data direct bool is_online_gather_statistics_; + int64_t ddl_task_id_; }; } // namespace observer } // namespace oceanbase diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index 535ee5357..610aa40b5 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -625,7 +625,7 @@ int ObCreateTableExecutor::execute(ObExecContext &ctx, ObCreateTableStmt &stmt) ret = OB_ERR_UNEXPECTED; LOG_WARN("session_info should not be nullptr", KR(ret)); } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish( - tenant_id, res.task_id_, session_info, common_rpc_proxy, true))) { + tenant_id, res.task_id_, false/*do not retry at executor*/, session_info, common_rpc_proxy, true))) { if (storage::ObMViewExecutorUtil::is_mview_refresh_retry_ret_code(ret)) { LOG_WARN("retry create mview", KR(ret), K(tenant_id), "task_id", res.task_id_); ret = OB_EAGAIN; @@ -813,7 +813,7 @@ int ObAlterTableExecutor::alter_table_rpc_v2( ObIArray &ddl_ress = res.ddl_res_array_; for (int64_t i = 0; OB_SUCC(ret) && i < ddl_ress.count(); ++i) { ObDDLRes &ddl_res = ddl_ress.at(i); - if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(ddl_res.tenant_id_, ddl_res.task_id_, my_session, common_rpc_proxy, is_support_cancel))) { + if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(ddl_res.tenant_id_, ddl_res.task_id_, false/*do not retry at executor*/, my_session, common_rpc_proxy, is_support_cancel))) { LOG_WARN("wait drop index finish", K(ret)); } } @@ -1142,7 +1142,7 @@ int ObAlterTableExecutor::execute(ObExecContext &ctx, ObAlterTableStmt &stmt) // do nothing, don't check if data is valid } else if (OB_FAIL(refresh_schema_for_table(tenant_id))) { LOG_WARN("refresh_schema_for_table failed", K(ret)); - } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, my_session, common_rpc_proxy))) { + } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, res.ddl_need_retry_at_executor_, my_session, common_rpc_proxy))) { LOG_WARN("wait check constraint finish", K(ret)); } } @@ -1153,7 +1153,7 @@ int ObAlterTableExecutor::execute(ObExecContext &ctx, ObAlterTableStmt &stmt) } else { if (OB_FAIL(refresh_schema_for_table(tenant_id))) { LOG_WARN("refresh_schema_for_table failed", K(ret)); - } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, my_session, common_rpc_proxy))) { + } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, res.ddl_need_retry_at_executor_, my_session, common_rpc_proxy))) { LOG_WARN("wait fk constraint finish", K(ret)); } } @@ -1166,7 +1166,7 @@ int ObAlterTableExecutor::execute(ObExecContext &ctx, ObAlterTableStmt &stmt) int64_t affected_rows = 0; if (OB_FAIL(refresh_schema_for_table(alter_table_arg.exec_tenant_id_))) { LOG_WARN("refresh_schema_for_table failed", K(ret)); - } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, my_session, common_rpc_proxy))) { + } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, res.ddl_need_retry_at_executor_, my_session, common_rpc_proxy))) { LOG_WARN("fail to wait ddl finish", K(ret), K(tenant_id), K(res)); } } diff --git a/src/sql/engine/expr/ob_expr_eval_functions.cpp b/src/sql/engine/expr/ob_expr_eval_functions.cpp index 2887daabb..b04638c24 100644 --- a/src/sql/engine/expr/ob_expr_eval_functions.cpp +++ b/src/sql/engine/expr/ob_expr_eval_functions.cpp @@ -347,6 +347,7 @@ #include "ob_expr_extract_cert_expired_time.h" #include "ob_expr_transaction_id.h" #include "ob_expr_inner_row_cmp_val.h" +#include "ob_expr_last_refresh_scn.h" #include "ob_expr_sql_udt_construct.h" #include "ob_expr_priv_attribute_access.h" #include "ob_expr_temp_table_ssid.h" @@ -1160,7 +1161,7 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { NULL, // ObExprGTIDSubtract::eval_subtract, /* 687 */ NULL, // ObExprWaitForExecutedGTIDSet::eval_wait_for_executed_gtid_set, /* 688 */ NULL, // ObExprWaitUntilSQLThreadAfterGTIDs::eval_wait_until_sql_thread_after_gtids /* 689 */ - NULL, // ObExprLastRefreshScn::eval_last_refresh_scn /* 690 */ + ObExprLastRefreshScn::eval_last_refresh_scn, /* 690 */ NULL, // ObExprDocLength::generate_doc_length, /*691*/ NULL, // ObExprTopNFilter::eval_topn_filter, /* 692 */ NULL, // ObExprIsEnabledRole::eval_is_enabled_role, /* 693 */ diff --git a/src/sql/engine/expr/ob_expr_extra_info_factory.cpp b/src/sql/engine/expr/ob_expr_extra_info_factory.cpp index 12e5fd10d..bba3747bf 100644 --- a/src/sql/engine/expr/ob_expr_extra_info_factory.cpp +++ b/src/sql/engine/expr/ob_expr_extra_info_factory.cpp @@ -33,6 +33,7 @@ #include "sql/engine/expr/ob_expr_sql_udt_construct.h" #include "sql/engine/expr/ob_expr_priv_attribute_access.h" #include "sql/engine/expr/ob_expr_lrpad.h" +#include "sql/engine/expr/ob_expr_last_refresh_scn.h" #include "sql/engine/expr/ob_expr_json_schema_valid.h" #include "sql/engine/expr/ob_expr_json_schema_validation_report.h" #include "sql/engine/expr/ob_expr_json_utils.h" @@ -108,6 +109,7 @@ void ObExprExtraInfoFactory::register_expr_extra_infos() REG_EXTRA_INFO(T_FUN_SYS_PRIV_SQL_UDT_ATTR_ACCESS, ObExprUdtAttrAccessInfo); REG_EXTRA_INFO(T_FUN_SYS_LPAD, ObExprOracleLRpadInfo); REG_EXTRA_INFO(T_FUN_SYS_RPAD, ObExprOracleLRpadInfo); + REG_EXTRA_INFO(T_FUN_SYS_LAST_REFRESH_SCN, ObExprLastRefreshScn::LastRefreshScnExtraInfo); REG_EXTRA_INFO(T_FUN_SYS_JSON_SCHEMA_VALID, ObExprJsonSchemaValidInfo); REG_EXTRA_INFO(T_FUN_SYS_JSON_SCHEMA_VALIDATION_REPORT, ObExprJsonSchemaValidInfo); REG_EXTRA_INFO(T_FUN_SYS_JSON_VALUE, ObExprJsonQueryParamInfo); diff --git a/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp b/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp new file mode 100644 index 000000000..c89a9cba1 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp @@ -0,0 +1,202 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "sql/engine/expr/ob_expr_last_refresh_scn.h" +#include "sql/engine/expr/ob_expr_util.h" +#include "sql/engine/ob_physical_plan_ctx.h" +#include "sql/engine/ob_exec_context.h" + +namespace oceanbase +{ +using namespace common; + +namespace sql +{ +ObExprLastRefreshScn::ObExprLastRefreshScn(ObIAllocator &alloc) + : ObFuncExprOperator(alloc, T_FUN_SYS_LAST_REFRESH_SCN, N_SYS_LAST_REFRESH_SCN, 0, NOT_VALID_FOR_GENERATED_COL, NOT_ROW_DIMENSION) +{ +} +ObExprLastRefreshScn::~ObExprLastRefreshScn() +{ +} + +int ObExprLastRefreshScn::calc_result_type0(ObExprResType &type, ObExprTypeCtx &type_ctx) const +{ + UNUSED(type_ctx); + if (is_oracle_mode()) { + const ObAccuracy &acc = ObAccuracy::DDL_DEFAULT_ACCURACY2[common::ORACLE_MODE][common::ObNumberType]; + type.set_number(); + type.set_scale(acc.get_scale()); + type.set_precision(acc.get_precision()); + } else { + const ObAccuracy &acc = common::ObAccuracy::DDL_DEFAULT_ACCURACY[common::ObUInt64Type]; + type.set_uint64(); + type.set_scale(acc.get_scale()); + type.set_precision(acc.get_precision()); + type.set_result_flag(NOT_NULL_FLAG); + } + return OB_SUCCESS; +} + +int ObExprLastRefreshScn::eval_last_refresh_scn(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; + uint64_t scn = OB_INVALID_SCN_VAL; + const ObExprLastRefreshScn::LastRefreshScnExtraInfo *info = + static_cast(expr.extra_info_); + const ObPhysicalPlanCtx *phy_plan_ctx = NULL; + if (OB_ISNULL(info) || OB_ISNULL(phy_plan_ctx = ctx.exec_ctx_.get_physical_plan_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("phy_plan_ctx is null", K(ret), K(info), K(phy_plan_ctx)); + } else if (OB_UNLIKELY(OB_INVALID_SCN_VAL == (scn = phy_plan_ctx->get_last_refresh_scn(info->mview_id_)))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("failed to get valid last_refresh_scn for mview id", K(ret), K(info->mview_id_), K(lbt())); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "materialized view id in last_refresh_scn"); + } else if (ObUInt64Type == expr.datum_meta_.type_) { + expr_datum.set_uint(scn); + } else { + ObNumStackOnceAlloc tmp_alloc; + number::ObNumber num; + if (OB_FAIL(num.from(scn, tmp_alloc))) { + LOG_WARN("copy number fail", K(ret)); + } else { + expr_datum.set_number(num); + } + } + return ret; +} + +int ObExprLastRefreshScn::cg_expr(ObExprCGCtx &op_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const +{ + int ret = OB_SUCCESS; + LastRefreshScnExtraInfo *extra_info = NULL; + void *buf = NULL; + if (OB_ISNULL(op_cg_ctx.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("allocator is null", K(ret)); + } else if (OB_ISNULL(buf = op_cg_ctx.allocator_->alloc(sizeof(LastRefreshScnExtraInfo)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else { + extra_info = new(buf) LastRefreshScnExtraInfo(*op_cg_ctx.allocator_, type_); + extra_info->mview_id_ = static_cast(raw_expr).get_mview_id(); + rt_expr.eval_func_ = ObExprLastRefreshScn::eval_last_refresh_scn; + rt_expr.extra_info_ = extra_info; + } + return OB_SUCCESS; +} + +OB_SERIALIZE_MEMBER(ObExprLastRefreshScn::LastRefreshScnExtraInfo, mview_id_); + +int ObExprLastRefreshScn::LastRefreshScnExtraInfo::deep_copy(common::ObIAllocator &allocator, + const ObExprOperatorType type, + ObIExprExtraInfo *&copied_info) const +{ + int ret = OB_SUCCESS; + OZ(ObExprExtraInfoFactory::alloc(allocator, type, copied_info)); + LastRefreshScnExtraInfo &other = *static_cast(copied_info); + if (OB_SUCC(ret)) { + other = *this; + } + return ret; +} + +int ObExprLastRefreshScn::set_last_refresh_scns(const ObIArray &src_mview_ids, + ObMySQLProxy *sql_proxy, + ObSQLSessionInfo *session, + const share::SCN &scn, + ObIArray &mview_ids, + ObIArray &last_refresh_scns) +{ + int ret = OB_SUCCESS; + mview_ids.reuse(); + last_refresh_scns.reuse(); + ObSEArray res_ids; + ObSEArray res_scns; + if (OB_ISNULL(sql_proxy) || OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString sql; + sqlclient::ObMySQLResult *mysql_result = NULL; + if (OB_FAIL(get_last_refresh_scn_sql(scn, src_mview_ids, sql))) { + LOG_WARN("failed to get last refresh scn sql", K(ret)); + } else if (OB_FAIL(sql_proxy->read(res, session->get_effective_tenant_id(), sql.ptr()))) { + LOG_WARN("execute sql failed", K(ret), K(sql)); + } else if (OB_ISNULL(mysql_result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("execute sql fail", K(ret)); + } else { + ObSEArray res_ids; + ObSEArray res_scns; + const int64_t col_idx0 = 0; + const int64_t col_idx1 = 1; + while (OB_SUCC(ret) && OB_SUCC(mysql_result->next())) { + int64_t mview_id = OB_INVALID_ID; + uint64_t last_refresh_scn = OB_INVALID_SCN_VAL; + if (OB_FAIL(mysql_result->get_int(col_idx0, mview_id)) + || OB_FAIL(mysql_result->get_uint(col_idx1, last_refresh_scn))) { + LOG_WARN("fail to get int/uint value", K(ret)); + } else if (OB_FAIL(res_ids.push_back(mview_id)) + || OB_FAIL(res_scns.push_back(last_refresh_scn))) { + LOG_WARN("fail to push back", K(ret)); + } + } + if (OB_UNLIKELY(OB_SUCCESS == ret || OB_ITER_END == ret) + && (OB_FAIL(mview_ids.assign(res_ids)) || OB_FAIL(last_refresh_scns.assign(res_scns)))) { + LOG_WARN("fail to assign array", K(ret)); + } + } + } + } + return ret; +} + +int ObExprLastRefreshScn::get_last_refresh_scn_sql(const share::SCN &scn, + const ObIArray &mview_ids, + ObSqlString &sql) +{ + int ret = OB_SUCCESS; + ObSqlString mview_id_array; + if (OB_UNLIKELY(mview_ids.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect empty array", K(ret), K(mview_ids)); + } else if (OB_UNLIKELY(mview_ids.count() > 100)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("more than 100 different materialized view id used in last_refresh_scn", K(ret), K(mview_ids.count())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "more than 100 different materialized view id used in last_refresh_scn is"); + } else { + for (int i = 0; OB_SUCC(ret) && i < mview_ids.count(); ++i) { + if (OB_FAIL(mview_id_array.append_fmt(0 == i ? "%ld" : ",%ld", mview_ids.at(i)))) { + LOG_WARN("fail to append fmt", KR(ret)); + } + } + } + if (OB_FAIL(ret)) { + } else if (SCN::invalid_scn() == scn) { + if (OB_FAIL(sql.assign_fmt("SELECT MVIEW_ID, LAST_REFRESH_SCN FROM `%s`.`%s` WHERE TENANT_ID = 0 AND MVIEW_ID IN (%.*s)", + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, + (int)mview_id_array.length(), mview_id_array.ptr()))) { + LOG_WARN("fail to assign sql", KR(ret)); + } + } else if (OB_FAIL(sql.assign_fmt("SELECT MVIEW_ID, LAST_REFRESH_SCN FROM `%s`.`%s` AS OF SNAPSHOT %ld WHERE TENANT_ID = 0 AND MVIEW_ID IN (%.*s)", + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, scn.get_val_for_sql(), + (int)mview_id_array.length(), mview_id_array.ptr()))) { + LOG_WARN("fail to assign sql", KR(ret), K(scn)); + } + return ret; +} + +} //namespace sql +} //namespace oceanbase diff --git a/src/sql/engine/expr/ob_expr_last_refresh_scn.h b/src/sql/engine/expr/ob_expr_last_refresh_scn.h new file mode 100644 index 000000000..19a0d8950 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_last_refresh_scn.h @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_OB_EXPR_LAST_REFRESH_SCN_H_ +#define OCEANBASE_SQL_OB_EXPR_LAST_REFRESH_SCN_H_ + +#include "sql/engine/expr/ob_expr_operator.h" + +namespace oceanbase +{ +namespace sql +{ +class ObExprLastRefreshScn : public ObFuncExprOperator +{ +public: + explicit ObExprLastRefreshScn(common::ObIAllocator &alloc); + virtual ~ObExprLastRefreshScn(); + + struct LastRefreshScnExtraInfo : public ObIExprExtraInfo + { + OB_UNIS_VERSION(1); + public: + LastRefreshScnExtraInfo(common::ObIAllocator &alloc, ObExprOperatorType type) + : ObIExprExtraInfo(alloc, type), + mview_id_(share::OB_INVALID_SCN_VAL) + { + } + virtual ~LastRefreshScnExtraInfo() { } + virtual int deep_copy(common::ObIAllocator &allocator, + const ObExprOperatorType type, + ObIExprExtraInfo *&copied_info) const override; + uint64_t mview_id_; + }; + + virtual int calc_result_type0(ObExprResType &type, common::ObExprTypeCtx &type_ctx) const; + static int eval_last_refresh_scn(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + + static int set_last_refresh_scns(const ObIArray &src_mview_ids, + ObMySQLProxy *sql_proxy, + ObSQLSessionInfo *session, + const share::SCN &scn, + ObIArray &mview_ids, + ObIArray &last_refresh_scns); + static int get_last_refresh_scn_sql(const share::SCN &scn, + const ObIArray &mview_ids, + ObSqlString &sql); + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObExprLastRefreshScn); +}; + +} //sql +} //oceanbase +#endif //OCEANBASE_SQL_OB_EXPR_CURRENT_SCN_H_ diff --git a/src/sql/engine/expr/ob_expr_operator_factory.cpp b/src/sql/engine/expr/ob_expr_operator_factory.cpp index b5c5b34d1..2c2e2d6f7 100644 --- a/src/sql/engine/expr/ob_expr_operator_factory.cpp +++ b/src/sql/engine/expr/ob_expr_operator_factory.cpp @@ -418,6 +418,7 @@ #include "sql/engine/expr/ob_expr_extract_cert_expired_time.h" #include "sql/engine/expr/ob_expr_transaction_id.h" #include "sql/engine/expr/ob_expr_inner_row_cmp_val.h" +#include "sql/engine/expr/ob_expr_last_refresh_scn.h" #include "sql/engine/expr/ob_expr_priv_st_makeenvelope.h" #include "sql/engine/expr/ob_expr_priv_st_clipbybox2d.h" #include "sql/engine/expr/ob_expr_priv_st_pointonsurface.h" @@ -1048,6 +1049,7 @@ void ObExprOperatorFactory::register_expr_operators() REG_OP(ObExprExtractExpiredTime); REG_OP(ObExprTransactionId); REG_OP(ObExprInnerRowCmpVal); + REG_OP(ObExprLastRefreshScn); // REG_OP(ObExprTopNFilter); REG_OP(ObExprPrivSTMakeEnvelope); REG_OP(ObExprPrivSTClipByBox2D); @@ -1382,6 +1384,7 @@ void ObExprOperatorFactory::register_expr_operators() REG_OP_ORCL(ObExprJsonObjectStar); REG_OP_ORCL(ObExprTransactionId); REG_OP_ORCL(ObExprInnerRowCmpVal); + REG_OP_ORCL(ObExprLastRefreshScn); // REG_OP_ORCL(ObExprTopNFilter); } diff --git a/src/sql/engine/ob_physical_plan.h b/src/sql/engine/ob_physical_plan.h index fb591a14d..c34672576 100644 --- a/src/sql/engine/ob_physical_plan.h +++ b/src/sql/engine/ob_physical_plan.h @@ -512,6 +512,8 @@ public: const ObSubSchemaCtx &get_subschema_ctx() const { return subschema_ctx_; } int set_all_local_session_vars(ObIArray *all_local_session_vars); ObIArray & get_all_local_session_vars() { return all_local_session_vars_; } + inline const ObIArray &get_mview_ids() const { return mview_ids_; } + int set_mview_ids(const ObIArray &mview_ids) { return mview_ids_.assign(mview_ids); } public: static const int64_t MAX_PRINTABLE_SIZE = 2 * 1024 * 1024; private: @@ -692,7 +694,6 @@ public: bool disable_auto_memory_mgr_; private: common::ObFixedArray all_local_session_vars_; - public: bool udf_has_dml_stmt_; private: diff --git a/src/sql/engine/ob_physical_plan_ctx.cpp b/src/sql/engine/ob_physical_plan_ctx.cpp index 97fbde0b9..d4d7d6e9d 100644 --- a/src/sql/engine/ob_physical_plan_ctx.cpp +++ b/src/sql/engine/ob_physical_plan_ctx.cpp @@ -1257,5 +1257,16 @@ int ObPhysicalPlanCtx::init_param_store_after_deserialize() return ret; } +uint64_t ObPhysicalPlanCtx::get_last_refresh_scn(uint64_t mview_id) const +{ + uint64_t last_refresh_scn = OB_INVALID_SCN_VAL; + for (int64_t i = 0; OB_INVALID_SCN_VAL == last_refresh_scn && i < mview_ids_.count(); ++i) { + if (mview_id == mview_ids_.at(i)) { + last_refresh_scn = last_refresh_scns_.at(i); + } + } + return last_refresh_scn; +} + } //sql } //oceanbase diff --git a/src/sql/engine/ob_physical_plan_ctx.h b/src/sql/engine/ob_physical_plan_ctx.h index f1c2bd061..6c9c196e3 100644 --- a/src/sql/engine/ob_physical_plan_ctx.h +++ b/src/sql/engine/ob_physical_plan_ctx.h @@ -476,6 +476,9 @@ public: ObIArray &get_array_param_groups() { return array_param_groups_; } int set_all_local_session_vars(ObIArray &all_local_session_vars); int get_local_session_vars(int64_t idx, const ObLocalSessionVar *&local_vars); + common::ObIArray &get_mview_ids() { return mview_ids_; } + common::ObIArray &get_last_refresh_scns() { return last_refresh_scns_; } + uint64_t get_last_refresh_scn(uint64_t mview_id) const; void set_tx_id(int64_t tx_id) { tx_id_ = tx_id; } int64_t get_tx_id() const { return tx_id_; } void set_tm_sessid(int64_t tm_sessid) { tm_sessid_ = tm_sessid; } diff --git a/src/sql/executor/ob_remote_executor_processor.cpp b/src/sql/executor/ob_remote_executor_processor.cpp index 1c831ee38..28a0fc058 100644 --- a/src/sql/executor/ob_remote_executor_processor.cpp +++ b/src/sql/executor/ob_remote_executor_processor.cpp @@ -30,6 +30,7 @@ #include "sql/ob_sql.h" #include "share/scheduler/ob_tenant_dag_scheduler.h" #include "storage/tx/ob_trans_service.h" +#include "sql/engine/expr/ob_expr_last_refresh_scn.h" namespace oceanbase { @@ -463,6 +464,14 @@ int ObRemoteBaseExecuteP::execute_remote_plan(ObExecContext &exec_ctx, LOG_WARN("created operator is NULL", K(ret)); } else if (OB_FAIL(plan_ctx->reserve_param_space(plan.get_param_count()))) { LOG_WARN("reserve rescan param space failed", K(ret), K(plan.get_param_count())); + } else if (!plan.get_mview_ids().empty() && plan_ctx->get_mview_ids().empty() + && OB_FAIL(ObExprLastRefreshScn::set_last_refresh_scns(plan.get_mview_ids(), + exec_ctx.get_sql_proxy(), + exec_ctx.get_my_session(), + exec_ctx.get_das_ctx().get_snapshot().core_.version_, + plan_ctx->get_mview_ids(), + plan_ctx->get_last_refresh_scns()))) { + LOG_WARN("fail to set last_refresh_scns", K(ret), K(plan.get_mview_ids())); } else { if (OB_FAIL(se_op->open())) { LOG_WARN("fail open task", K(ret)); diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index 427d04d17..615e7510f 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -52,6 +52,7 @@ #include "storage/tx/ob_xa_ctx.h" #include "sql/engine/dml/ob_link_op.h" #include +#include "sql/engine/expr/ob_expr_last_refresh_scn.h" using namespace oceanbase::sql; using namespace oceanbase::common; @@ -588,6 +589,14 @@ OB_INLINE int ObResultSet::do_open_plan(ObExecContext &ctx) if (OB_FAIL(ret)) { } else if (OB_FAIL(start_stmt())) { LOG_WARN("fail start stmt", K(ret)); + } else if (!physical_plan_->get_mview_ids().empty() && OB_PHY_PLAN_REMOTE != physical_plan_->get_plan_type() + && OB_FAIL(ObExprLastRefreshScn::set_last_refresh_scns(physical_plan_->get_mview_ids(), + ctx.get_sql_proxy(), + ctx.get_my_session(), + ctx.get_das_ctx().get_snapshot().core_.version_, + ctx.get_physical_plan_ctx()->get_mview_ids(), + ctx.get_physical_plan_ctx()->get_last_refresh_scns()))) { + LOG_WARN("fail to set last_refresh_scns", K(ret), K(physical_plan_->get_mview_ids())); } else { /* 将exec_result_设置到executor的运行时环境中，用于返回数据 */ /* 执行plan, diff --git a/src/sql/optimizer/ob_del_upd_log_plan.cpp b/src/sql/optimizer/ob_del_upd_log_plan.cpp index c19e02ced..d0051c265 100644 --- a/src/sql/optimizer/ob_del_upd_log_plan.cpp +++ b/src/sql/optimizer/ob_del_upd_log_plan.cpp @@ -23,6 +23,7 @@ #include "sql/rewrite/ob_transform_utils.h" #include "sql/dblink/ob_dblink_utils.h" #include "sql/engine/cmd/ob_table_direct_insert_service.h" +#include "sql/session/ob_sql_session_info.h" using namespace oceanbase; using namespace sql; @@ -69,7 +70,7 @@ int ObDelUpdLogPlan::compute_dml_parallel() LOG_WARN("get unexpected parallel", K(ret), K(dml_parallel), K(opt_ctx.get_parallel_rule())); } else { max_dml_parallel_ = dml_parallel; - use_pdml_ = (opt_ctx.is_online_ddl() || + use_pdml_ = (opt_ctx.is_online_ddl() || session_info->get_ddl_info().is_mview_complete_refresh() || (ObGlobalHint::DEFAULT_PARALLEL < dml_parallel && is_strict_mode(session_info->get_sql_mode()))); } diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 6462d243b..d5398e20b 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -142,6 +142,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"completion", COMPLETION}, {"compressed", COMPRESSED}, {"compression", COMPRESSION}, + {"computation", COMPUTATION}, {"compute", COMPUTE}, {"concurrent", CONCURRENT}, {"condensed", CONDENSED}, @@ -413,6 +414,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"lag", LAG}, {"language", LANGUAGE}, {"last", LAST}, + {"last_refresh_scn", LAST_REFRESH_SCN}, {"last_value", LAST_VALUE}, {"lateral", LATERAL}, {"lead", LEAD}, @@ -721,6 +723,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"return", RETURN}, {"returns", RETURNS}, {"reverse", REVERSE}, + {"rewrite", REWRITE}, {"revoke", REVOKE}, {"right", RIGHT}, {"rlike", REGEXP}, diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 30ad0e8f3..27c4652cd 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -1020,6 +1020,8 @@ Timestamp{whitespace}?\"[^\"]*\" { LEADING { return LEADING_HINT; } ORDERED { return ORDERED; } NO_REWRITE { return NO_REWRITE; } +MV_REWRITE { return MV_REWRITE; } +NO_MV_REWRITE { return NO_MV_REWRITE; } FULL { return FULL_HINT; } USE_MERGE { return USE_MERGE; } NO_USE_MERGE { return NO_USE_MERGE; } diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 6b23582b9..b6a0b1cdb 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -176,6 +176,7 @@ PROJECT_PRUNE NO_PROJECT_PRUNE SIMPLIFY_SET NO_SIMPLIFY_SET OUTER_TO_INNER NO_OU COALESCE_SQ NO_COALESCE_SQ COUNT_TO_EXISTS NO_COUNT_TO_EXISTS LEFT_TO_ANTI NO_LEFT_TO_ANTI ELIMINATE_JOIN NO_ELIMINATE_JOIN PUSH_LIMIT NO_PUSH_LIMIT PULLUP_EXPR NO_PULLUP_EXPR WIN_MAGIC NO_WIN_MAGIC AGGR_FIRST_UNNEST NO_AGGR_FIRST_UNNEST JOIN_FIRST_UNNEST NO_JOIN_FIRST_UNNEST +MV_REWRITE NO_MV_REWRITE DECORRELATE NO_DECORRELATE // optimize hint INDEX_HINT FULL_HINT NO_INDEX_HINT USE_DAS_HINT NO_USE_DAS_HINT @@ -272,7 +273,7 @@ END_P SET_VAR DELIMITER CACHE CALIBRATION CALIBRATION_INFO CANCEL CASCADED CAST CATALOG_NAME CHAIN CHANGED CHARSET CHECKSUM CHECKPOINT CHUNK CIPHER CLASS_ORIGIN CLEAN CLEAR CLIENT CLONE CLOG CLOSE CLUSTER CLUSTER_ID CLUSTER_NAME COALESCE COLUMN_STAT CODE COLLATION COLUMN_FORMAT COLUMN_NAME COLUMNS COMMENT COMMIT COMMITTED COMPACT COMPLETION COMPLETE - COMPRESSED COMPRESSION COMPUTE CONCURRENT CONDENSED CONNECTION CONSISTENT CONSISTENT_MODE CONSTRAINT_CATALOG + COMPRESSED COMPRESSION COMPUTATION COMPUTE CONCURRENT CONDENSED CONNECTION CONSISTENT CONSISTENT_MODE CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA CONTAINS CONTEXT CONTRIBUTORS COPY COUNT CPU CREATE_TIMESTAMP CTXCAT CTX_ID CUBE CURDATE CURRENT STACKED CURTIME CURSOR_NAME CUME_DIST CYCLE CALC_PARTITION_ID CONNECT @@ -302,7 +303,8 @@ END_P SET_VAR DELIMITER KEY_BLOCK_SIZE KEY_VERSION KVCACHE KV_ATTRIBUTES - LAG LANGUAGE LAST LAST_VALUE LATERAL LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_ + LAG LANGUAGE LAST LAST_REFRESH_SCN LAST_VALUE LATERAL LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_ + LISTAGG LOB_INROW_THRESHOLD LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS LEVEL LN LOG LS LINK LOG_RESTORE_SOURCE LINE_DELIMITER @@ -336,7 +338,7 @@ END_P SET_VAR DELIMITER REBUILD RECOVER RECOVERY_WINDOW RECYCLE REDO_BUFFER_SIZE REDOFILE REDUNDANCY REDUNDANT REFRESH REGION RELAY RELAYLOG RELAY_LOG_FILE RELAY_LOG_POS RELAY_THREAD RELOAD REMAP REMOVE REORGANIZE REPAIR REPEATABLE REPLICA REPLICA_NUM REPLICA_TYPE REPLICATION REPORT RESET RESOURCE RESOURCE_POOL RESOURCE_POOL_LIST RESPECT RESTART - RESTORE RESUME RETURNED_SQLSTATE RETURNS RETURNING REVERSE ROLLBACK ROLLUP ROOT + RESTORE RESUME RETURNED_SQLSTATE RETURNS RETURNING REVERSE REWRITE ROLLBACK ROLLUP ROOT ROOTTABLE ROOTSERVICE ROOTSERVICE_LIST ROUTINE ROW ROLLING ROWID ROW_COUNT ROW_FORMAT ROWS RTREE RUN RECYCLEBIN ROTATE ROW_NUMBER RUDUNDANT RECURSIVE RANDOM REDO_TRANSPORT_OPTIONS REMOTE_OSS RT RANK READ_ONLY RECOVERY REJECT ROLE @@ -418,7 +420,7 @@ END_P SET_VAR DELIMITER %type sort_list sort_key opt_asc_desc sort_list_for_group_by sort_key_for_group_by opt_asc_desc_for_group_by opt_column_id %type opt_query_expression_option_list query_expression_option_list query_expression_option opt_distinct opt_distinct_or_all opt_separator projection %type from_list table_references table_reference table_factor normal_relation_factor dot_relation_factor relation_factor -%type relation_factor_in_hint relation_factor_in_hint_list relation_factor_in_pq_hint opt_relation_factor_in_hint_list relation_factor_in_use_join_hint_list +%type relation_factor_in_hint relation_factor_in_hint_list relation_factor_in_pq_hint opt_relation_factor_in_hint_list relation_factor_in_use_join_hint_list relation_factor_in_mv_hint_list opt_relation_factor_in_mv_hint_list %type relation_factor_in_leading_hint_list joined_table tbl_name table_subquery table_subquery_alias %type relation_factor_with_star relation_with_star_list opt_with_star %type index_hint_type key_or_index index_hint_scope index_element index_list opt_index_list @@ -474,9 +476,9 @@ END_P SET_VAR DELIMITER %type drop_index_stmt hint_options opt_expr_as_list expr_as_list expr_with_opt_alias substr_params opt_comma substr_or_substring %type /*frozen_type*/ opt_binary %type ip_port -%type create_view_stmt view_name opt_column_list opt_table_id opt_tablet_id view_select_stmt opt_check_option opt_tablet_id_no_empty -%type create_mview_stmt create_mview_refresh mv_refresh_on_clause mv_refresh_mode mv_refresh_interval mv_start_clause mv_next_clause -%type mv_refresh_method +%type create_view_stmt view_name opt_column_list opt_mv_column_list mv_column_list opt_table_id opt_tablet_id view_select_stmt opt_check_option opt_tablet_id_no_empty +%type create_mview_stmt create_mview_opts mview_refresh_opt mv_refresh_on_clause mv_refresh_mode mv_refresh_interval mv_start_clause mv_next_clause +%type mv_refresh_method mview_enable_disable %type name_list %type partition_role ls_role zone_desc opt_zone_desc server_or_zone opt_server_or_zone opt_partitions opt_subpartitions add_or_alter_zone_options alter_or_change_or_modify %type ls opt_tenant_list_or_ls_or_tablet_id ls_server_or_server_or_zone_or_tenant add_or_alter_zone_option @@ -3075,6 +3077,10 @@ MOD '(' expr ',' expr ')' malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_ST_ASMVT, 5, $3, $5, $7, $9, $11); $$->reserved_ = 0; } +| LAST_REFRESH_SCN '(' INTNUM ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_LAST_REFRESH_SCN, 1, $3); +} | SUM_OPNSIZE '(' expr ')' { malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SUM_OPNSIZE, 2, NULL, $3); @@ -8100,7 +8106,8 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW * *****************************************************************************/ create_mview_stmt: -create_with_opt_hint MATERIALIZED VIEW view_name opt_column_list opt_table_option_list opt_partition_option create_mview_refresh AS view_select_stmt opt_check_option +create_with_opt_hint MATERIALIZED VIEW view_name opt_mv_column_list opt_table_option_list opt_partition_option create_mview_opts +AS view_select_stmt opt_check_option { ParseNode *table_options = NULL; merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); @@ -8114,7 +8121,7 @@ create_with_opt_hint MATERIALIZED VIEW view_name opt_column_list opt_table_optio NULL, $11, /* with option */ NULL, /* force view opt */ - $8, + $8, /* mview options */ $7, /* partition option */ table_options, /* table options */ $1 /* hint */ @@ -8124,7 +8131,46 @@ create_with_opt_hint MATERIALIZED VIEW view_name opt_column_list opt_table_optio } ; -create_mview_refresh: +create_mview_opts: +mview_refresh_opt +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); + $$->value_ = 0; +} +| mview_refresh_opt mview_enable_disable ON QUERY COMPUTATION +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); + $$->value_ = $2[0]; +} +| mview_refresh_opt mview_enable_disable QUERY REWRITE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); + $$->value_ = $2[0] << 1; +} +| mview_refresh_opt mview_enable_disable ON QUERY COMPUTATION mview_enable_disable QUERY REWRITE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); + $$->value_ = $2[0] | ($6[0] << 1); +} +| mview_refresh_opt mview_enable_disable QUERY REWRITE mview_enable_disable ON QUERY COMPUTATION +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_OPTIONS, 1, $1); + $$->value_ = $5[0] | ($2[0] << 1); +} +; + +mview_enable_disable: +DISABLE +{ + $$[0] = 0; +} +| ENABLE +{ + $$[0] = 1; +} +; + +mview_refresh_opt: REFRESH mv_refresh_method mv_refresh_on_clause mv_refresh_interval { malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REFRESH_INFO, 2, @@ -8295,6 +8341,35 @@ relation_factor { $$ = $1; } ; +opt_mv_column_list: +'(' mv_column_list ')' +{ + merge_nodes($$, result, T_COLUMN_LIST, $2); +} +| /*EMPTY*/ { $$ = NULL; } +; + +mv_column_list: +column_name_list +{ + $$ = $1; +} +| column_name_list ',' PRIMARY KEY opt_index_using_algorithm '(' column_name_list ')' opt_index_using_algorithm opt_comment +{ + ParseNode *col_list= NULL; + ParseNode *pk_node = NULL; + merge_nodes(col_list, result, T_COLUMN_LIST, $7); + malloc_non_terminal_node(pk_node, result->malloc_pool_, T_PRIMARY_KEY, 3, col_list, NULL != $9 ? $9 : $5, $10); + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, pk_node); +} +| PRIMARY KEY opt_index_using_algorithm '(' column_name_list ')' opt_index_using_algorithm opt_comment +{ + ParseNode *col_list= NULL; + merge_nodes(col_list, result, T_COLUMN_LIST, $5); + malloc_non_terminal_node($$, result->malloc_pool_, T_PRIMARY_KEY, 3, col_list, NULL != $7 ? $7 : $3, $8); +} +; + opt_column_list: '(' column_name_list ')' { @@ -10215,6 +10290,18 @@ NO_REWRITE opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_NO_REWRITE, 1, $2); } +| MV_REWRITE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REWRITE, 2, NULL, NULL); +} +| MV_REWRITE '(' qb_name_option opt_relation_factor_in_mv_hint_list ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_REWRITE, 2, $3, $4); +} +| NO_MV_REWRITE opt_qb_name +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_MV_NO_REWRITE, 1, $2); +} | MERGE_HINT opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_MERGE_HINT, 2, $2, NULL); @@ -12355,6 +12442,30 @@ relation_sep_option: {} ; +relation_factor_in_mv_hint_list: +normal_relation_factor +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_RELATION_FACTOR_IN_HINT, 2, $1, NULL); +} +| relation_factor_in_mv_hint_list relation_sep_option normal_relation_factor +{ + ParseNode *mock_rel_in_hint_node = NULL; + malloc_non_terminal_node(mock_rel_in_hint_node, result->malloc_pool_, T_RELATION_FACTOR_IN_HINT, 2, $3, NULL); + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, mock_rel_in_hint_node); +} +; + +opt_relation_factor_in_mv_hint_list: +/* EMPTY */ +{ + $$ = NULL; +} +| relation_factor_in_mv_hint_list +{ + merge_nodes($$, result, T_RELATION_FACTOR_IN_HINT_LIST, $1); +} +; + opt_relation_factor_in_hint_list: /* EMPTY */ { @@ -20520,6 +20631,7 @@ ACCOUNT | COMPLETION | COMPRESSED | COMPRESSION +| COMPUTATION | COMPUTE | CONCURRENT | CONDENSED @@ -20701,6 +20813,7 @@ ACCOUNT | LATERAL %prec LOWER_PARENS | LANGUAGE | LAST +| LAST_REFRESH_SCN | LAST_VALUE | LEAD | LEADER @@ -20935,6 +21048,7 @@ ACCOUNT | RETURNING | RETURNS | REVERSE +| REWRITE | ROLE | ROLLBACK | ROLLING diff --git a/src/sql/printer/ob_raw_expr_printer.cpp b/src/sql/printer/ob_raw_expr_printer.cpp index d809ec8ef..0882ae6a0 100644 --- a/src/sql/printer/ob_raw_expr_printer.cpp +++ b/src/sql/printer/ob_raw_expr_printer.cpp @@ -3304,6 +3304,10 @@ int ObRawExprPrinter::print(ObSysFunRawExpr *expr) PRINT_EXPR(expr->get_param_expr(2)); break; } + case T_FUN_SYS_LAST_REFRESH_SCN: { + DATA_PRINTF("%.*s(%ld)", LEN_AND_PTR(func_name), expr->get_mview_id()); + break; + } // for bugfix: 52438113/52226266 case T_FUN_SYS_PRIV_SQL_UDT_CONSTRUCT: { OZ(print_sql_udt_construct(expr)); diff --git a/src/sql/resolver/ddl/ob_create_index_resolver.cpp b/src/sql/resolver/ddl/ob_create_index_resolver.cpp index 9d7d0c2e0..6fa502198 100644 --- a/src/sql/resolver/ddl/ob_create_index_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_index_resolver.cpp @@ -258,6 +258,14 @@ int ObCreateIndexResolver::resolve_index_column_node( } } + if (OB_SUCC(ret) && cnt_func_index) { + if (OB_UNLIKELY(tbl_schema->mv_container_table())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("use functional index on materialized view not supported", K(ret), KPC(tbl_schema)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use functional index on materialized view"); + } + } + // In oracle mode, we need to check if the new index is on the same cols with old indexes. CHECK_COMPATIBILITY_MODE(session_info_); if (OB_SUCC(ret) && lib::is_oracle_mode()) { diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index 53227602c..c8dbd3f6b 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -90,76 +90,33 @@ int ObCreateTableResolver::add_primary_key_part(const ObString &column_name, { int ret = OB_SUCCESS; ObCreateTableStmt *create_table_stmt = static_cast(stmt_); - bool is_oracle_mode = lib::is_oracle_mode(); - if (OB_ISNULL(session_info_) || OB_ISNULL(create_table_stmt)) { + ObColumnSchemaV2 *col = NULL; + if (OB_ISNULL(create_table_stmt)) { ret = OB_NOT_INIT; - SQL_RESV_LOG(WARN, "session or stmt is null", KP(session_info_), KP(create_table_stmt), K(ret)); - } else if (static_cast(table_id_) > 0 - && OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id( - session_info_->get_effective_tenant_id(), table_id_, is_oracle_mode))) { - LOG_WARN("fail to check oracle mode", KR(ret), K_(table_id)); - } else { - ObColumnSchemaV2 *col = NULL; - ObTableSchema &table_schema = create_table_stmt->get_create_table_arg().schema_; - col = table_schema.get_column_schema(column_name); - if (OB_ISNULL(col)) { - ret = OB_ERR_KEY_COLUMN_DOES_NOT_EXITS; - LOG_USER_ERROR(OB_ERR_KEY_COLUMN_DOES_NOT_EXITS, column_name.length(), column_name.ptr()); - SQL_RESV_LOG(WARN, "column '%s' does not exists", K(ret), K(to_cstring(column_name))); - } else if (OB_FAIL(check_add_column_as_pk_allowed(*col))) { - LOG_WARN("the column can not be primary key", K(ret)); - } else { - int64_t index = -1; - bool is_found = false; - for (int64_t i = 0; OB_SUCC(ret) && !is_found && i < stats.count(); ++i) { - if (stats.at(i).column_id_ == col->get_column_id()) { - index = i; - is_found = true; - } - } - if (-1 == index) { - ret = OB_ERR_UNEXPECTED; - SQL_RESV_LOG(WARN, "fail to find column stat", K(ret), K(column_name)); - } else if (!is_oracle_mode) { - // mysql 模式下，建表时主键列被 set null 或被 set default value = null，都要报错 - // oracle 模式下，建表时主键列被 set null 或被 set default value = null，都不会报错，所以跳过下面这个检查 - if (stats.at(index).is_set_null_ - || (stats.at(index).is_set_default_value_ && col->get_cur_default_value().is_null())) { - ret = OB_ERR_PRIMARY_CANT_HAVE_NULL; - - } - } else { /*do nothing*/ } - } - if (OB_SUCC(ret)) { - if (col->get_rowkey_position() > 0) { - ret = OB_ERR_COLUMN_DUPLICATE; - LOG_USER_ERROR(OB_ERR_COLUMN_DUPLICATE, column_name.length(), column_name.ptr()); - } else if (OB_USER_MAX_ROWKEY_COLUMN_NUMBER == primary_keys_.count()) { - ret = OB_ERR_TOO_MANY_ROWKEY_COLUMNS; - LOG_USER_ERROR(OB_ERR_TOO_MANY_ROWKEY_COLUMNS, OB_USER_MAX_ROWKEY_COLUMN_NUMBER); - } else if (col->is_string_type()) { - int64_t length = 0; - if (OB_FAIL(col->get_byte_length(length, is_oracle_mode, false))) { - SQL_RESV_LOG(WARN, "fail to get byte length of column", KR(ret), K(is_oracle_mode)); - } else if ((pk_data_length += length) > OB_MAX_USER_ROW_KEY_LENGTH) { - ret = OB_ERR_TOO_LONG_KEY_LENGTH; - LOG_USER_ERROR(OB_ERR_TOO_LONG_KEY_LENGTH, OB_MAX_USER_ROW_KEY_LENGTH); - } else if (length <= 0) { - ret = OB_ERR_WRONG_KEY_COLUMN; - LOG_USER_ERROR(OB_ERR_WRONG_KEY_COLUMN, column_name.length(), column_name.ptr()); - } else { - // do nothing - } + SQL_RESV_LOG(WARN, "stmt is null", KP(create_table_stmt), K(ret)); + } else if (OB_FAIL(ObCreateTableResolverBase::add_primary_key_part(column_name, + create_table_stmt->get_create_table_arg().schema_, + primary_keys_.count(), + pk_data_length, + col))) { + LOG_WARN("failed to add primary key part", KR(ret), K(column_name)); + } else if (OB_FAIL(primary_keys_.push_back(col->get_column_id()))) { + SQL_RESV_LOG(WARN, "push primary key to array failed", K(ret)); + } else if (!lib::is_oracle_mode()) { + // mysql 模式下，建表时主键列被 set null 或被 set default value = null，都要报错 + // oracle 模式下，建表时主键列被 set null 或被 set default value = null，都不会报错，所以跳过下面这个检查 + ObColumnResolveStat *stat = NULL; + for (int64_t i = 0; NULL == stat && OB_SUCC(ret) && i < stats.count(); ++i) { + if (stats.at(i).column_id_ == col->get_column_id()) { + stat = &stats.at(i); } } - if (OB_SUCC(ret)) { - if (OB_FAIL(primary_keys_.push_back(col->get_column_id()))) { - SQL_RESV_LOG(WARN, "push primary key to array failed", K(ret)); - } else { - col->set_rowkey_position(primary_keys_.count()); - col->set_nullable(false); - ret = table_schema.set_rowkey_info(*col); - } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(stat)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "fail to find column stat", K(ret), K(column_name)); + } else if (stat->is_set_null_ || (stat->is_set_default_value_ && col->get_cur_default_value().is_null())) { + ret = OB_ERR_PRIMARY_CANT_HAVE_NULL; } } return ret; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index 70a64c652..c59d9d868 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -400,5 +400,52 @@ int ObCreateTableResolverBase::set_table_option_to_schema(ObTableSchema &table_s return ret; } +int ObCreateTableResolverBase::add_primary_key_part(const ObString &column_name, + ObTableSchema &table_schema, + const int64_t cur_rowkey_size, + int64_t &pk_data_length, + ObColumnSchemaV2 *&col) +{ + int ret = OB_SUCCESS; + col = NULL; + bool is_oracle_mode = lib::is_oracle_mode(); + int64_t length = 0; + if (OB_ISNULL(session_info_)) { + ret = OB_NOT_INIT; + SQL_RESV_LOG(WARN, "session is null", KP(session_info_), K(ret)); + } else if (static_cast(table_id_) > 0 + && OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id( + session_info_->get_effective_tenant_id(), table_id_, is_oracle_mode))) { + LOG_WARN("fail to check oracle mode", KR(ret), K_(table_id)); + } else if (OB_ISNULL(col = table_schema.get_column_schema(column_name))) { + ret = OB_ERR_KEY_COLUMN_DOES_NOT_EXITS; + LOG_USER_ERROR(OB_ERR_KEY_COLUMN_DOES_NOT_EXITS, column_name.length(), column_name.ptr()); + SQL_RESV_LOG(WARN, "column '%s' does not exists", K(ret), K(to_cstring(column_name))); + } else if (OB_FAIL(check_add_column_as_pk_allowed(*col))) { + LOG_WARN("the column can not be primary key", K(ret)); + } else if (col->get_rowkey_position() > 0) { + ret = OB_ERR_COLUMN_DUPLICATE; + LOG_USER_ERROR(OB_ERR_COLUMN_DUPLICATE, column_name.length(), column_name.ptr()); + } else if (OB_USER_MAX_ROWKEY_COLUMN_NUMBER == cur_rowkey_size) { + ret = OB_ERR_TOO_MANY_ROWKEY_COLUMNS; + LOG_USER_ERROR(OB_ERR_TOO_MANY_ROWKEY_COLUMNS, OB_USER_MAX_ROWKEY_COLUMN_NUMBER); + } else if (OB_FALSE_IT(col->set_nullable(false)) + || OB_FALSE_IT(col->set_rowkey_position(cur_rowkey_size + 1))) { + } else if (OB_FAIL(table_schema.set_rowkey_info(*col))) { + LOG_WARN("failed to set rowkey info", K(ret)); + } else if (!col->is_string_type()) { + /* do nothing */ + } else if (OB_FAIL(col->get_byte_length(length, is_oracle_mode, false))) { + SQL_RESV_LOG(WARN, "fail to get byte length of column", KR(ret), K(is_oracle_mode)); + } else if ((pk_data_length += length) > OB_MAX_USER_ROW_KEY_LENGTH) { + ret = OB_ERR_TOO_LONG_KEY_LENGTH; + LOG_USER_ERROR(OB_ERR_TOO_LONG_KEY_LENGTH, OB_MAX_USER_ROW_KEY_LENGTH); + } else if (length <= 0) { + ret = OB_ERR_WRONG_KEY_COLUMN; + LOG_USER_ERROR(OB_ERR_WRONG_KEY_COLUMN, column_name.length(), column_name.ptr()); + } + return ret; +} + }//end namespace sql }//end namespace oceanbase diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.h b/src/sql/resolver/ddl/ob_create_table_resolver_base.h index 83124682c..532918e3e 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.h @@ -38,6 +38,11 @@ protected: share::schema::ObTableSchema &table_schema, const bool is_partition_option_node_with_opt); int set_table_option_to_schema(share::schema::ObTableSchema &table_schema); + int add_primary_key_part(const ObString &column_name, + ObTableSchema &table_schema, + const int64_t cur_rowkey_size, + int64_t &pk_data_length, + ObColumnSchemaV2 *&col); }; diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.cpp b/src/sql/resolver/ddl/ob_create_view_resolver.cpp index 543813164..303fde21f 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_view_resolver.cpp @@ -26,6 +26,7 @@ #include "storage/mview/ob_mview_sched_job_utils.h" #include "sql/resolver/mv/ob_mv_checker.h" #include "observer/virtual_table/ob_table_columns.h" +#include "sql/rewrite/ob_transformer_impl.h" namespace oceanbase { @@ -130,6 +131,7 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) ObArray column_list; ObArray comment_list; bool has_dblink_node = false; + ParseNode *mv_primary_key_node = NULL; share::schema::ObSchemaGetterGuard *schema_guard = NULL; uint64_t database_id = OB_INVALID_ID; ObString old_database_name; @@ -279,8 +281,12 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) } if (OB_FAIL(ret)) { } else if (OB_FAIL(resolve_column_list(view_columns_node, - column_list))) { + column_list, + mv_primary_key_node))) { LOG_WARN("fail to resolve view columns", K(ret)); + } else if (OB_UNLIKELY(!is_materialized_view && NULL != mv_primary_key_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected primary key node for non materialized view", K(ret)); } else if (OB_ISNULL(select_stmt = view_table_resolver.get_select_stmt())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(view_table_resolver.get_select_stmt())); @@ -372,69 +378,33 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) } } - if (is_materialized_view) { //container table is only for mv - if (OB_SUCC(ret)) { - if (OB_FAIL(ObResolverUtils::check_schema_valid_for_mview(table_schema))) { - LOG_WARN("failed to check schema valid for mview", KR(ret), K(table_schema)); - } else if (OB_FAIL(resolve_table_options(parse_tree.children_[TABLE_OPTION_NODE], false))) { - LOG_WARN("fail to resolve table options", KR(ret)); - } - } - - SMART_VAR(ObMVAdditionalInfo, mv_ainfo) { - ObTableSchema &container_table_schema = mv_ainfo.container_table_schema_; - if (OB_SUCC(ret)) { - if (OB_FAIL(container_table_schema.assign(table_schema))) { - LOG_WARN("fail to assign table schema", KR(ret)); - } else { - container_table_schema.set_table_type(ObTableType::USER_TABLE); - container_table_schema.get_view_schema().reset(); - container_table_schema.set_max_dependency_version(OB_INVALID_VERSION); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(add_hidden_tablet_seq_col(container_table_schema))) { - LOG_WARN("fail to add hidden pk", KR(ret)); - } - } - if (OB_SUCC(ret)) { - pctfree_ = 0; // set default pctfree value for non-sys table - if (OB_FAIL(resolve_partition_option(parse_tree.children_[PARTITION_NODE], container_table_schema, true))) { - LOG_WARN("fail to resolve_partition_option", KR(ret)); - } else if (OB_FAIL(set_table_option_to_schema(container_table_schema))) { - SQL_RESV_LOG(WARN, "set table option to schema failed", KR(ret)); - } else { - container_table_schema.set_collation_type(collation_type_); - container_table_schema.set_charset_type(charset_type_); - container_table_schema.set_table_pk_mode(TPKM_TABLET_SEQ_PK); //非主键表需要设置这个 - container_table_schema.set_table_organization_mode(TOM_HEAP_ORGANIZED); - - container_table_schema.set_mv_container_table(IS_MV_CONTAINER_TABLE); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(resolve_hints(parse_tree.children_[HINT_NODE], *stmt, container_table_schema))) { - LOG_WARN("resolve hints failed", K(ret)); - } else { - mv_ainfo.mv_refresh_info_.parallel_ = stmt->get_parallelism(); - } - } - if (OB_SUCC(ret)) { - ObViewSchema &view_schema = table_schema.get_view_schema(); - if (OB_FAIL(resolve_mv_refresh_info(parse_tree.children_[MVIEW_NODE], mv_ainfo.mv_refresh_info_))) { - LOG_WARN("fail to resolve mv refresh info", KR(ret)); - } else if (ObMVRefreshMethod::FAST == mv_ainfo.mv_refresh_info_.refresh_method_ - && OB_FAIL(ObMVChecker::check_mv_fast_refresh_valid(select_stmt, - params_.expr_factory_, - params_.session_info_))) { - LOG_WARN("fail to check fast refresh valid", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(create_arg.mv_ainfo_.push_back(mv_ainfo))) { - LOG_WARN("fail to push back container table schema", KR(ret)); - } - } + if (OB_SUCC(ret) && is_materialized_view) { + ObMVAdditionalInfo *mv_ainfo = NULL; + ObCreateTableStmt *create_table_stmt = static_cast(stmt_); + ObSEArray &csts = create_table_stmt->get_create_table_arg().constraint_list_; + if (OB_FAIL(ObResolverUtils::check_schema_valid_for_mview(table_schema))) { + LOG_WARN("failed to check schema valid for mview", KR(ret), K(table_schema)); + } else if (OB_FAIL(resolve_table_options(parse_tree.children_[TABLE_OPTION_NODE], false))) { + LOG_WARN("fail to resolve table options", KR(ret)); + } else if (OB_ISNULL(mv_ainfo = create_arg.mv_ainfo_.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate ObMVAdditionalInfo from array error", K(ret)); + } else if (OB_FAIL(mv_ainfo->container_table_schema_.assign(table_schema))) { + LOG_WARN("fail to assign table schema", KR(ret)); + } else if (OB_FAIL(resolve_materialized_view_container_table(parse_tree.children_[PARTITION_NODE], + mv_primary_key_node, + mv_ainfo->container_table_schema_, + csts))) { + LOG_WARN("fail do resolve for materialized view", K(ret)); + } else if (OB_FAIL(resolve_mv_options(select_stmt, + parse_tree.children_[MVIEW_NODE], + mv_ainfo->mv_refresh_info_, + table_schema))) { + LOG_WARN("fail to resolve mv options", K(ret)); + } else if (OB_FAIL(resolve_hints(parse_tree.children_[HINT_NODE], *stmt, mv_ainfo->container_table_schema_))) { + LOG_WARN("resolve hints failed", K(ret)); + } else { + mv_ainfo->mv_refresh_info_.parallel_ = stmt->get_parallelism(); } } @@ -502,6 +472,91 @@ int ObCreateViewResolver::try_add_error_info(const uint64_t error_number, return ret; } +int ObCreateViewResolver::resolve_materialized_view_container_table(ParseNode *partition_node, + ParseNode *mv_primary_key_node, + ObTableSchema &container_table_schema, + ObSEArray& csts) +{ + int ret = OB_SUCCESS; + container_table_schema.set_table_type(ObTableType::USER_TABLE); + container_table_schema.get_view_schema().reset(); + container_table_schema.set_max_dependency_version(OB_INVALID_VERSION); + pctfree_ = 0; // set default pctfree value for non-sys table + if (OB_FAIL(resolve_partition_option(partition_node, container_table_schema, true))) { + LOG_WARN("fail to resolve_partition_option", KR(ret)); + } else if (OB_FAIL(set_table_option_to_schema(container_table_schema))) { + SQL_RESV_LOG(WARN, "set table option to schema failed", KR(ret)); + } else if (NULL != mv_primary_key_node + && OB_FAIL(resolve_primary_key_node(*mv_primary_key_node, container_table_schema))) { + LOG_WARN("failed to resolve primary key node", K(ret)); + } else if (0 < container_table_schema.get_rowkey_column_num()) { // create mv with primary key + container_table_schema.set_table_pk_mode(ObTablePKMode::TPKM_OLD_NO_PK); + container_table_schema.set_table_organization_mode(ObTableOrganizationMode::TOM_INDEX_ORGANIZED); + if (is_oracle_mode() && OB_FAIL(resolve_pk_constraint_node(*mv_primary_key_node, ObString::make_empty_string(), csts))) { + LOG_WARN("failed to add pk constraint for oracle mode", KR(ret)); + } + } else if (OB_FAIL(add_hidden_tablet_seq_col(container_table_schema))) { + LOG_WARN("fail to add hidden pk", KR(ret)); + } else { // create mv without primary key + container_table_schema.set_table_pk_mode(TPKM_TABLET_SEQ_PK); + container_table_schema.set_table_organization_mode(TOM_HEAP_ORGANIZED); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(container_table_schema.check_primary_key_cover_partition_column())) { + SQL_RESV_LOG(WARN, "fail to check primary key cover partition column", KR(ret)); + } else { + container_table_schema.set_collation_type(collation_type_); + container_table_schema.set_charset_type(charset_type_); + container_table_schema.set_mv_container_table(IS_MV_CONTAINER_TABLE); + } + return ret; +} + +int ObCreateViewResolver::resolve_primary_key_node(ParseNode &pk_node, + ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + ParseNode *cur_node = NULL; + if (OB_UNLIKELY(2 > pk_node.num_child_) || OB_ISNULL(cur_node = pk_node.children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(pk_node.num_child_), K(cur_node)); + } else if (OB_UNLIKELY(T_COLUMN_LIST != cur_node->type_ || cur_node->num_child_ <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected node", K(ret), K(get_type_name(cur_node->type_)), K(cur_node->num_child_)); + } else { + ParseNode *key_node = NULL; + int64_t pk_data_length = 0; + ObColumnSchemaV2 *col = NULL; + for (int32_t i = 0; OB_SUCC(ret) && i < cur_node->num_child_; ++i) { + if (OB_ISNULL(key_node = cur_node->children_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i)); + } else if (OB_FAIL(ObCreateTableResolverBase::add_primary_key_part(ObString(key_node->str_len_, key_node->str_value_), + table_schema, i, + pk_data_length, col))) { + LOG_WARN("failed to add primary key part", K(ret), K(i)); + } + } + if (OB_FAIL(ret) || is_oracle_mode()) { + } else if (OB_UNLIKELY(3 != pk_node.num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(pk_node.num_child_)); + } else { + if (NULL != (cur_node = pk_node.children_[1])) { + table_schema.set_index_using_type(T_USING_HASH == cur_node->type_ ? share::schema::USING_HASH + : share::schema::USING_BTREE); + } + if (NULL != (cur_node = pk_node.children_[2])) { + if (OB_FAIL(table_schema.set_pk_comment(ObString(cur_node->str_len_, cur_node->str_value_)))) { + LOG_WARN("fail to set primary key comment", K(ret), K(ObString(cur_node->str_len_, cur_node->str_value_))); + } + } + } + } + return ret; +} + int ObCreateViewResolver::check_view_columns(ObSelectStmt &select_stmt, ParseNode *view_columns_node, share::schema::ObErrorInfo &error_info, @@ -519,39 +574,46 @@ int ObCreateViewResolver::check_view_columns(ObSelectStmt &select_stmt, ObString dup_col_name; hash::ObHashSet view_col_names; int64_t select_item_size = select_stmt.get_select_item_size(); - if (NULL != view_columns_node && view_columns_node->num_child_ > 0) { - ObCollationType cs_type = CS_TYPE_INVALID; - if (OB_UNLIKELY(!is_force_view && select_item_size != view_columns_node->num_child_)) { - ret = OB_ERR_VIEW_WRONG_LIST; - LOG_WARN("view columns is not equal with select columns", K(select_item_size), - K(view_columns_node->num_child_)); - } else if (OB_FAIL(session_info_->get_collation_connection(cs_type))) { - LOG_WARN("fail to get collation_connection", K(ret)); - } else if (OB_FAIL(view_col_names.create(view_columns_node->num_child_))) { - LOG_WARN("failed to init hashset", K(ret), K(select_stmt.get_select_items().count())); - } else if (is_force_view && select_item_size != view_columns_node->num_child_) { - if (OB_FAIL(try_add_error_info(OB_ERR_VIEW_WRONG_LIST, error_info))) { - LOG_WARN("failed to add error info to for force view", K(ret)); - } else { - can_expand_star = false; - add_undefined_columns = true; - LOG_TRACE("force view columns is not equal with select columns", K(select_item_size), - K(view_columns_node->num_child_)); - } + bool has_view_columns_node = false; + ObCollationType cs_type = CS_TYPE_INVALID; + if ((!is_force_view || select_item_size > 0) + && OB_FAIL(view_col_names.create(select_item_size))) { + LOG_WARN("failed to init hashset", K(ret), K(select_item_size)); + } else if (NULL == view_columns_node || view_columns_node->num_child_ <= 0) { + /* do nothing */ + } else if (is_force_view && select_item_size != view_columns_node->num_child_) { + has_view_columns_node = true; + if (OB_FAIL(try_add_error_info(OB_ERR_VIEW_WRONG_LIST, error_info))) { + LOG_WARN("failed to add error info to for force view", K(ret)); + } else { + can_expand_star = false; + add_undefined_columns = true; + LOG_TRACE("force view columns is not equal with select columns", K(select_item_size), + K(view_columns_node->num_child_)); } + } else if (OB_FAIL(session_info_->get_collation_connection(cs_type))) { + LOG_WARN("fail to get collation_connection", K(ret)); + } else { + ParseNode *child_node = NULL; + int64_t col_cnt_from_node = 0; for (int64_t i = 0; OB_SUCC(ret) && !is_col_dup && i < view_columns_node->num_child_; i++) { - if (OB_ISNULL(view_columns_node->children_[i])) { + if (OB_ISNULL(child_node = view_columns_node->children_[i])) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid null children", K(ret), K(view_columns_node->children_[i])); - } else if (FALSE_IT(dup_col_name = ObString::make_string(view_columns_node->children_[i]->str_value_))) { + LOG_WARN("invalid null children", K(ret), K(i), K(child_node)); + } else if (T_PRIMARY_KEY == child_node->type_) { + /* do nothing */ + } else if (FALSE_IT(dup_col_name = ObString::make_string(child_node->str_value_))) { } else if (OB_FAIL(ObCharset::tolower(cs_type, dup_col_name, dup_col_name, *allocator_))) { LOG_WARN("fail to lower string", K(ret)); } else if (OB_HASH_EXIST == (ret = view_col_names.set_refactored(dup_col_name, 0))) { + ++col_cnt_from_node; is_col_dup = true; ret = OB_SUCCESS; } else if (OB_FAIL(ret)) { LOG_WARN("failed to set hashset", K(ret)); - } else { /* do nothing */ } + } else { + ++col_cnt_from_node; + } } if (OB_SUCC(ret) && lib::is_oracle_mode() && can_expand_star && select_item_size > 0) { hash::ObHashSet select_item_names; @@ -570,6 +632,17 @@ int ObCreateViewResolver::check_view_columns(ObSelectStmt &select_stmt, } } } + if (OB_FAIL(ret) || 0 == col_cnt_from_node) { + } else if (OB_UNLIKELY(select_item_size != col_cnt_from_node)) { + ret = OB_ERR_VIEW_WRONG_LIST; + LOG_WARN("view columns is not equal with select columns", K(select_item_size), + K(col_cnt_from_node), K(view_columns_node->num_child_)); + } else { + has_view_columns_node = true; + } + } + + if (OB_FAIL(ret) || has_view_columns_node) { } else if (OB_UNLIKELY(is_force_view && 0 == select_item_size)) { if (OB_FAIL(try_add_error_info(OB_ERR_ONLY_HAVE_INVISIBLE_COL_IN_TABLE, error_info))) { LOG_WARN("failed to add error info to for force view", K(ret)); @@ -577,8 +650,6 @@ int ObCreateViewResolver::check_view_columns(ObSelectStmt &select_stmt, LOG_TRACE("force view must have at least one column that is not invisible", K(OB_ERR_ONLY_HAVE_INVISIBLE_COL_IN_TABLE)); } - } else if (OB_FAIL(view_col_names.create(select_item_size))) { - LOG_WARN("failed to init hashset", K(ret), K(select_item_size)); } else if (lib::is_oracle_mode()) { for (int64_t i = 0; OB_SUCC(ret) && !is_col_dup && i < select_stmt.get_select_items().count(); i++) { @@ -1179,6 +1250,58 @@ int ObCreateViewResolver::create_alias_names_auto( return ret; } +int ObCreateViewResolver::resolve_mv_options(const ObSelectStmt *stmt, + ParseNode *options_node, + ObMVRefreshInfo &refresh_info, + ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + refresh_info.refresh_method_ = ObMVRefreshMethod::FORCE; //default method is force + refresh_info.refresh_mode_ = ObMVRefreshMode::DEMAND; //default mode is demand + if (NULL == options_node) { + /* do nothing */ + } else if (OB_UNLIKELY(T_MV_OPTIONS != options_node->type_ || 1 != options_node->num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(options_node)); + } else if (OB_FAIL(resolve_mv_refresh_info(options_node->children_[0], refresh_info))) { + LOG_WARN("fail to resolve mv refresh info", KR(ret)); + } else { + const int64_t on_query_computation_flag = 1; + const int64_t query_rewrite_flag = 1 << 1; + if (options_node->value_ & on_query_computation_flag) { + table_schema.set_mv_on_query_computation(ObMVOnQueryComputationFlag::IS_MV_ON_QUERY_COMPUTATION); + } + if (options_node->value_ & query_rewrite_flag) { + table_schema.set_mv_enable_query_rewrite(ObMVEnableQueryRewriteFlag::IS_MV_ENABLE_QUERY_REWRITE); + } + } + if (OB_FAIL(ret)) { + } else if ((table_schema.mv_on_query_computation() || ObMVRefreshMethod::FAST == refresh_info.refresh_method_) + && OB_FAIL(ObMVChecker::check_mv_fast_refresh_valid(stmt, params_.stmt_factory_, + params_.expr_factory_, + params_.session_info_))) { + LOG_WARN("fail to check fast refresh valid", K(ret)); + } else if (table_schema.mv_on_query_computation() + && OB_FAIL(check_on_query_computation_supported(stmt))) { + LOG_WARN("fail to check on query computation mv column type", K(ret)); + } + return ret; +} + +int ObCreateViewResolver::check_on_query_computation_supported(const ObSelectStmt *stmt) +{ + int ret = OB_SUCCESS; + ObTransformerImpl::StmtFunc func; + if (OB_FAIL(ObTransformerImpl::check_stmt_functions(stmt, func))) { + LOG_WARN("failed to check stmt functions", K(ret)); + } else if (OB_UNLIKELY(func.contain_enum_set_values_)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "on query computation mview use enum type"); + LOG_WARN("not support on query computation mview use enum type", KR(ret), K(func.contain_enum_set_values_)); + } + return ret; +} + int ObCreateViewResolver::resolve_mv_refresh_info(ParseNode *refresh_info_node, ObMVRefreshInfo &refresh_info) { @@ -1186,9 +1309,6 @@ int ObCreateViewResolver::resolve_mv_refresh_info(ParseNode *refresh_info_node, if (allocator_ == nullptr) { ret = OB_ERR_UNEXPECTED; LOG_WARN("allocator_ is null", KR(ret)); - } else { - refresh_info.refresh_method_ = ObMVRefreshMethod::FORCE; //default method is force - refresh_info.refresh_mode_ = ObMVRefreshMode::DEMAND; //default mode is demand } char buf[OB_MAX_PROC_ENV_LENGTH]; int64_t pos = 0; @@ -1309,9 +1429,11 @@ int ObCreateViewResolver::resolve_mv_refresh_info(ParseNode *refresh_info_node, } int ObCreateViewResolver::resolve_column_list(ParseNode *view_columns_node, - ObIArray &column_list) + ObIArray &column_list, + ParseNode *&mv_primary_key_node) { int ret = OB_SUCCESS; + mv_primary_key_node = NULL; if (OB_ISNULL(allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("allocator_ is NULL", K(ret)); @@ -1326,6 +1448,13 @@ int ObCreateViewResolver::resolve_column_list(ParseNode *view_columns_node, if (OB_ISNULL(column_node)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column node should not be NULL", K(ret)); + } else if (T_PRIMARY_KEY == column_node->type_) { + if (OB_UNLIKELY(NULL != mv_primary_key_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("more than one primary key node", K(ret)); + } else { + mv_primary_key_node = column_node; + } } else { column.reset(); ObString column_name; diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.h b/src/sql/resolver/ddl/ob_create_view_resolver.h index 2c83e7d7d..e1c633068 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.h +++ b/src/sql/resolver/ddl/ob_create_view_resolver.h @@ -92,7 +92,12 @@ private: int check_privilege(ObCreateTableStmt *stmt, ObSelectStmt *select_stmt); int resolve_column_list(ParseNode *view_columns_node, - common::ObIArray &column_list); + common::ObIArray &column_list, + ParseNode *&mv_primary_key_node); + int resolve_mv_options(const ObSelectStmt *stmt, + ParseNode *options_node, + ObMVRefreshInfo &refresh_info, + ObTableSchema &table_schema); int resolve_mv_refresh_info(ParseNode *refresh_info_node, ObMVRefreshInfo &refresh_info); @@ -132,7 +137,12 @@ private: hash::ObHashMap &select_tables, hash::ObHashMap &any_tables); int add_hidden_tablet_seq_col(ObTableSchema &table_schema); - + int resolve_materialized_view_container_table(ParseNode *partition_node, + ParseNode *mv_primary_key_node, + ObTableSchema &container_table_schema, + ObSEArray& csts); + int resolve_primary_key_node(ParseNode &pk_node, ObTableSchema &table_schema); + int check_on_query_computation_supported(const ObSelectStmt *stmt); private: DISALLOW_COPY_AND_ASSIGN(ObCreateViewResolver); }; diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index e6c4b07ff..2688039d4 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -5577,32 +5577,26 @@ int ObDMLResolver::resolve_base_or_alias_table_item_normal(uint64_t tenant_id, //the feature only use in mysqtest case, not open for user const ObTableSchema *tab_schema = NULL; item->is_index_table_ = tschema->is_index_table(); - - if (tschema->is_materialized_view()) { - item->ref_id_ = tschema->get_data_table_id(); - item->table_type_ = tschema->get_table_type(); - } else { - item->ref_id_ = tschema->get_table_id(); - } item->table_id_ = generate_table_id(); item->type_ = TableItem::ALIAS_TABLE; //主表schema if (OB_FAIL(schema_checker_->get_table_schema(session_info_->get_effective_tenant_id(), tschema->get_data_table_id(), tab_schema))) { LOG_WARN("get data table schema failed", K(ret), K_(item->ref_id)); } else { - item->table_name_ = tab_schema->get_table_name_str(); //主表的名字 - if (alias_name.length() == 0) { - if (tschema->is_materialized_view()) { - if (!synonym_name.empty()) { - item->alias_name_ = synonym_name; //mv可能有同义词，需要考虑同义词 - } else { - item->alias_name_ = tschema->get_table_name_str(); //将mv作为主表的alias name - } - } else { - item->alias_name_ = tschema->get_table_name_str(); //将索引名作为主表的alias name - } + if (tschema->is_materialized_view()) { + item->ref_id_ = tschema->get_data_table_id(); + item->mview_id_ = tschema->get_table_id(); + item->table_type_ = tschema->get_table_type(); + item->need_expand_rt_mv_ = !params_.is_for_rt_mv_ && tschema->mv_on_query_computation(); + item->table_name_ = tschema->get_table_name_str(); + item->alias_name_ = alias_name.empty() ? tschema->get_table_name_str() : alias_name; + LOG_DEBUG("resolve mv table", K(ret), K(tschema->get_table_name()), K(params_.is_for_rt_mv_), + K(tschema->mv_enable_query_rewrite()), K(tschema->mv_on_query_computation())); } else { - item->alias_name_ = alias_name; + item->ref_id_ = tschema->get_table_id(); + item->table_name_ = tab_schema->get_table_name_str(); //主表的名字 + //将索引名作为主表的alias name + item->alias_name_ = alias_name.empty() ? tschema->get_table_name_str() : alias_name; } //如果是查索引表，需要将主表的依赖也要加入到plan中 ObSchemaObjVersion table_version; @@ -14210,7 +14204,14 @@ int ObDMLResolver::resolve_transform_hint(const ParseNode &hint_node, case T_PLACE_GROUP_BY: case T_NO_PLACE_GROUP_BY: { if (OB_FAIL(resolve_place_group_by_hint(hint_node, trans_hint))) { - LOG_WARN("failed to resolve win magic hint", K(ret)); + LOG_WARN("failed to resolve place group by hint", K(ret)); + } + break; + } + case T_MV_REWRITE: + case T_MV_NO_REWRITE: { + if (OB_FAIL(resolve_mv_rewrite_hint(hint_node, trans_hint))) { + LOG_WARN("failed to resolve mv rewrite hint", K(ret)); } break; } @@ -15126,6 +15127,32 @@ int ObDMLResolver::resolve_place_group_by_hint(const ParseNode &hint_node, return ret; } +int ObDMLResolver::resolve_mv_rewrite_hint(const ParseNode &hint_node, + ObTransHint *&hint) +{ + int ret = OB_SUCCESS; + hint = NULL; + ObMVRewriteHint *mv_rewrite_hint = NULL; + ObString qb_name; + if (OB_UNLIKELY(1 != hint_node.num_child_ && 2 != hint_node.num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected mv rewrite hint", K(ret), K(hint_node.num_child_)); + } else if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, mv_rewrite_hint))) { + LOG_WARN("failed to create eliminate join hint", K(ret)); + } else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) { + LOG_WARN("Failed to resolve qb name node", K(ret)); + } else if (2 == hint_node.num_child_ && NULL != hint_node.children_[1] && + OB_FAIL(resolve_simple_table_list_in_hint(hint_node.children_[1], + mv_rewrite_hint->get_mv_list()))) { + LOG_WARN("failed to resovle simple table list in hint", K(ret)); + } else { + mv_rewrite_hint->set_qb_name(qb_name); + hint = mv_rewrite_hint; + LOG_DEBUG("show mv_rewrite_hint hint", KPC(mv_rewrite_hint)); + } + return ret; +} + int ObDMLResolver::resolve_tb_name_list(const ParseNode *tb_name_list_node, ObIArray> &tb_name_list) { diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 152369bad..2cee4f017 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -924,6 +924,7 @@ private: int resolve_eliminate_join_hint(const ParseNode &hint_node, ObTransHint *&hint); int resolve_win_magic_hint(const ParseNode &hint_node, ObTransHint *&hint); int resolve_place_group_by_hint(const ParseNode &hint_node, ObTransHint *&hint); + int resolve_mv_rewrite_hint(const ParseNode &hint_node, ObTransHint *&hint); int resolve_tb_name_list(const ParseNode *tb_name_list_node, ObIArray> &tb_name_list); int resolve_alloc_ops(const ParseNode &alloc_op_node, ObIArray &alloc_op_hints); int resolve_tables_in_leading_hint(const ParseNode *tables_node, ObLeadingTable &leading_table); diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index a923906b6..bdceb1f58 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -256,7 +256,8 @@ int TableItem::deep_copy(ObIRawExprCopier &expr_copier, for_update_ = other.for_update_; for_update_wait_us_ = other.for_update_wait_us_; skip_locked_ = other.skip_locked_; - mock_id_ = other.mock_id_; + need_expand_rt_mv_ = other.need_expand_rt_mv_; + mview_id_ = other.mview_id_; node_ = other.node_; // should deep copy ? seems to be unnecessary flashback_query_type_ = other.flashback_query_type_; // dblink @@ -1836,7 +1837,8 @@ int ObDMLStmt::formalize_relation_exprs(ObSQLSessionInfo *session_info) } int ObDMLStmt::formalize_stmt_expr_reference(ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info) + ObSQLSessionInfo *session_info, + bool explicit_for_col /* default false */) { int ret = OB_SUCCESS; ObSEArray stmt_exprs; @@ -1892,7 +1894,7 @@ int ObDMLStmt::formalize_stmt_expr_reference(ObRawExprFactory *expr_factory, } else { /*do nothing*/ } } if (OB_SUCC(ret)) { - if (OB_FAIL(remove_useless_sharable_expr(expr_factory, session_info))) { + if (OB_FAIL(remove_useless_sharable_expr(expr_factory, session_info, explicit_for_col))) { LOG_WARN("failed to remove useless sharable expr", K(ret)); } else if (OB_FAIL(check_pseudo_column_valid())) { LOG_WARN("failed to check pseudo column", K(ret)); @@ -2081,37 +2083,41 @@ int ObDMLStmt::generated_column_depend_column_is_referred(ObRawExpr *expr, bool } int ObDMLStmt::remove_useless_sharable_expr(ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info) + ObSQLSessionInfo *session_info, + bool explicit_for_col) { int ret = OB_SUCCESS; UNUSED(expr_factory); UNUSED(session_info); for (int64_t i = column_items_.count() - 1; OB_SUCC(ret) && i >= 0; i--) { ObColumnRefRawExpr *expr = NULL; + bool is_referred = false; + bool need_remove = false; if (OB_ISNULL(expr = column_items_.at(i).expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (expr->is_explicited_reference() || expr->is_rowkey_column() - || expr->is_spatial_generated_column()) { - /*do nothing*/ + } else if (expr->is_explicited_reference()) { + need_remove = false; + } else if (explicit_for_col) { + need_remove = true; + } else if (expr->is_rowkey_column() || expr->is_spatial_generated_column()) { + need_remove = false; + } else if (OB_FAIL(is_referred_by_partitioning_expr(expr, is_referred))) { + LOG_WARN("failed to check whether is referred by partitioning expr", K(ret)); + } else if (is_referred) { + need_remove = false; + } else if (OB_FAIL(generated_column_depend_column_is_referred(expr, is_referred))) { + //if the generate column's depend expr has columns referred. + LOG_WARN("generated expr has normal column depends", K(ret)); } else { - bool is_referred = true; - if (OB_FAIL(is_referred_by_partitioning_expr(expr, is_referred))) { - LOG_WARN("failed to check whether is referred by partitioning expr", K(ret)); - } + need_remove = !is_referred; + } - if (OB_FAIL(ret) || is_referred) { - //if the generate column's depend expr has columns referred. - } else if (OB_FAIL(generated_column_depend_column_is_referred(expr, is_referred))) { - LOG_WARN("generated expr has normal column depends", K(ret)); - } - - if (OB_SUCC(ret) && !is_referred) { - if (OB_FAIL(column_items_.remove(i))) { - LOG_WARN("failed to remove column item", K(ret)); - } else { - LOG_TRACE("succeed to remove column items", K(expr), K(lbt())); - } + if (OB_SUCC(ret) && need_remove) { + if (OB_FAIL(column_items_.remove(i))) { + LOG_WARN("failed to remove column item", K(ret)); + } else { + LOG_TRACE("succeed to remove column items", K(expr), K(lbt())); } } } @@ -4630,14 +4636,14 @@ int ObDMLStmt::disable_writing_external_table(bool basic_stmt_is_dml /* defualt return ret; } -int ObDMLStmt::disable_writing_materialized_view() +int ObDMLStmt::disable_writing_materialized_view() const { int ret = OB_SUCCESS; bool disable_write_table = false; const TableItem *table_item = NULL; if (is_dml_write_stmt()) { - ObSEArray dml_table_infos; - if (OB_FAIL(static_cast(this)->get_dml_table_infos(dml_table_infos))) { + ObSEArray dml_table_infos; + if (OB_FAIL(static_cast(this)->get_dml_table_infos(dml_table_infos))) { LOG_WARN("failed to get dml table infos"); } for (int64_t i = 0; OB_SUCC(ret) && !disable_write_table && i < dml_table_infos.count(); ++i) { diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index 551182093..29c72692a 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -227,7 +227,8 @@ struct TableItem for_update_ = false; for_update_wait_us_ = -1; skip_locked_ = false; - mock_id_ = common::OB_INVALID_ID; + need_expand_rt_mv_ = false; + mview_id_ = common::OB_INVALID_ID; node_ = NULL; view_base_item_ = NULL; flashback_query_expr_ = nullptr; @@ -254,7 +255,6 @@ struct TableItem N_FOR_UPDATE, for_update_, N_WAIT, for_update_wait_us_, K_(skip_locked), - N_MOCK_ID, mock_id_, "view_base_item", (NULL == view_base_item_ ? OB_INVALID_ID : view_base_item_->table_id_), K_(dblink_id), K_(dblink_name), K_(link_database_name), K_(is_reverse_link), @@ -262,8 +262,7 @@ struct TableItem K_(is_view_table), K_(part_ids), K_(part_names), K_(cte_type), KPC_(function_table_expr), K_(flashback_query_type), KPC_(flashback_query_expr), K_(table_type), - K(table_values_), - K_(exec_params)); + K(table_values_), K_(exec_params), K_(mview_id), K_(need_expand_rt_mv)); enum TableType { @@ -381,8 +380,8 @@ struct TableItem bool for_update_; int64_t for_update_wait_us_;//0 means nowait, -1 means infinite bool skip_locked_; - //在hierarchical query, 记录由当前table_item mock出来的table_item的table id - uint64_t mock_id_; + bool need_expand_rt_mv_; // for real-time materialized view + uint64_t mview_id_; // for materialized view, ref_id_ is mv container table id, mview_id_ is the view id const ParseNode* node_; // base table item for updatable view const TableItem *view_base_item_; // seems to be useful only in the resolve phase @@ -821,14 +820,17 @@ public: int pull_all_expr_relation_id(); int formalize_stmt(ObSQLSessionInfo *session_info); int formalize_relation_exprs(ObSQLSessionInfo *session_info); - int formalize_stmt_expr_reference(ObRawExprFactory *expr_factory, ObSQLSessionInfo *session_info); + int formalize_stmt_expr_reference(ObRawExprFactory *expr_factory, + ObSQLSessionInfo *session_info, + bool explicit_for_col = false); int formalize_child_stmt_expr_reference(ObRawExprFactory *expr_factory, ObSQLSessionInfo *session_info); int set_sharable_expr_reference(ObRawExpr &expr, ExplicitedRefType ref_type); int check_pseudo_column_valid(); int get_ora_rowscn_column(const uint64_t table_id, ObPseudoColumnRawExpr *&ora_rowscn); virtual int remove_useless_sharable_expr(ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info); + ObSQLSessionInfo *session_info, + bool explicit_for_col); virtual int clear_sharable_expr_reference(); virtual int get_from_subquery_stmts(common::ObIArray &child_stmts) const; virtual int get_subquery_stmts(common::ObIArray &child_stmts) const; @@ -1163,7 +1165,7 @@ public: int check_has_subquery_in_function_table(bool &has_subquery_in_function_table) const; int disable_writing_external_table(bool basic_stmt_is_dml = false); - int disable_writing_materialized_view(); + int disable_writing_materialized_view() const; int formalize_query_ref_exprs(); int formalize_query_ref_exec_params(ObStmtExecParamFormatter &formatter, @@ -1178,6 +1180,10 @@ public: int do_formalize_lateral_derived_table_pre(); + int deep_copy_join_tables(ObIAllocator &allocator, + ObIRawExprCopier &expr_copier, + const ObDMLStmt &other); + int do_formalize_lateral_derived_table_post(); protected: @@ -1185,9 +1191,6 @@ protected: //获取到stmt中所有查询相关的表达式(由查询语句中指定的属性生成的表达式)的root expr protected: - int deep_copy_join_tables(ObIAllocator &allocator, - ObIRawExprCopier &expr_copier, - const ObDMLStmt &other); int construct_join_table(const ObDMLStmt &other, const JoinedTable &other_joined_table, JoinedTable &joined_table); diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index 3db84f131..a57681bb5 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -974,6 +974,7 @@ ObItemType ObHint::get_hint_type(ObItemType type) case T_NO_AGGR_FIRST_UNNEST: return T_AGGR_FIRST_UNNEST; case T_NO_JOIN_FIRST_UNNEST: return T_JOIN_FIRST_UNNEST; case T_NO_DECORRELATE : return T_DECORRELATE; + case T_MV_NO_REWRITE: return T_MV_REWRITE; // optimize hint case T_NO_USE_DAS_HINT: return T_USE_DAS_HINT; @@ -1031,6 +1032,7 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul case T_AGGR_FIRST_UNNEST: return is_enable_hint ? "AGGR_FIRST_UNNEST" : "NO_AGGR_FIRST_UNNEST"; case T_JOIN_FIRST_UNNEST: return is_enable_hint ? "JOIN_FIRST_UNNEST" : "NO_JOIN_FIRST_UNNEST"; case T_DECORRELATE : return is_enable_hint ? "DECORRELATE" : "NO_DECORRELATE"; + case T_MV_REWRITE: return is_enable_hint ? "MV_REWRITE" : "NO_MV_REWRITE"; // optimize hint case T_INDEX_HINT: return "INDEX"; case T_FULL_HINT: return "FULL"; @@ -1908,6 +1910,56 @@ bool ObCoalesceSqHint::has_qb_name_list(const ObIArray & qb_names) con return bret; } +int ObMVRewriteHint::assign(const ObMVRewriteHint &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObTransHint::assign(other))) { + LOG_WARN("fail to assign hint", K(ret)); + } else if (OB_FAIL(mv_list_.assign(other.mv_list_))) { + LOG_WARN("failed to assign mv list", K(ret)); + } + return ret; +} + +int ObMVRewriteHint::print_hint_desc(PlanText &plan_text) const +{ + int ret = OB_SUCCESS; + if (!mv_list_.empty()) { + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + for (int64_t i = 0; OB_SUCC(ret) && i < mv_list_.count(); ++i) { + if (i > 0 && OB_FAIL(BUF_PRINTF(", "))) { + LOG_WARN("fail to print comma", K(ret)); + } else if (OB_FAIL(mv_list_.at(i).print_table_in_hint(plan_text, true))) { + LOG_WARN("fail to print mv table", K(ret)); + } + } + } + return ret; +} + +int ObMVRewriteHint::check_mv_match_hint(ObCollationType cs_type, + const ObTableSchema *mv_schema, + const ObDatabaseSchema *db_schema, + bool &is_match) const +{ + int ret = OB_SUCCESS; + is_match = false; + if (OB_ISNULL(mv_schema) || OB_ISNULL(db_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_schema), K(db_schema)); + } else if (mv_list_.empty()) { + is_match = true; + } + for (int64_t i = 0; OB_SUCC(ret) && !is_match && i < mv_list_.count(); ++i) { + const ObTableInHint &table_in_hint = mv_list_.at(i); + is_match = 0 == ObCharset::strcmp(cs_type, table_in_hint.table_name_, mv_schema->get_table_name()) && + (table_in_hint.db_name_.empty() || 0 == ObCharset::strcmp(cs_type, table_in_hint.db_name_, db_schema->get_database_name_str())); + } + return ret; +} + int ObTableParallelHint::assign(const ObTableParallelHint &other) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index e996282cd..8710f366e 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -439,6 +439,7 @@ public: HINT_ELIMINATE_JOIN, HINT_GROUPBY_PLACEMENT, HINT_WIN_MAGIC, + HINT_MV_REWRITE, // optimize hint below HINT_OPTIMIZE, // normal optimize hint HINT_ACCESS_PATH, @@ -815,6 +816,32 @@ private: common::ObSEArray qb_name_list_; }; +class ObMVRewriteHint : public ObTransHint +{ +public: + ObMVRewriteHint(ObItemType hint_type) + : ObTransHint(hint_type), + mv_list_() + { + set_hint_class(HINT_MV_REWRITE); + } + int assign(const ObMVRewriteHint &other); + virtual ~ObMVRewriteHint() {} + + virtual int print_hint_desc(PlanText &plan_text) const override; + common::ObIArray &get_mv_list() { return mv_list_; } + const common::ObIArray &get_mv_list() const { return mv_list_; } + int check_mv_match_hint(ObCollationType cs_type, + const ObTableSchema *mv_schema, + const ObDatabaseSchema *db_schema, + bool &is_match) const; + + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(mv_list)); + +private: + common::ObSEArray mv_list_; +}; + class ObIndexHint : public ObOptHint { public: diff --git a/src/sql/resolver/dml/ob_select_stmt.cpp b/src/sql/resolver/dml/ob_select_stmt.cpp index b29262a85..b25afd028 100644 --- a/src/sql/resolver/dml/ob_select_stmt.cpp +++ b/src/sql/resolver/dml/ob_select_stmt.cpp @@ -156,11 +156,11 @@ int ObSelectStmt::check_aggr_and_winfunc(ObRawExpr &expr) if (expr.is_aggr_expr() && !ObRawExprUtils::find_expr(agg_items_, &expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("aggr expr does not exist in the stmt", K(ret)); + LOG_WARN("aggr expr does not exist in the stmt", K(ret), K(expr)); } else if (expr.is_win_func_expr() && !ObRawExprUtils::find_expr(win_func_exprs_, &expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("win func expr does not exist in the stmt", K(ret)); + LOG_WARN("win func expr does not exist in the stmt", K(ret), K(expr)); } return ret; } @@ -229,6 +229,7 @@ int ObSelectStmt::assign(const ObSelectStmt &other) is_hierarchical_query_ = other.is_hierarchical_query_; has_prior_ = other.has_prior_; has_reverse_link_ = other.has_reverse_link_; + is_expanded_mview_ = other.is_expanded_mview_; } return ret; } @@ -320,6 +321,7 @@ int ObSelectStmt::deep_copy_stmt_struct(ObIAllocator &allocator, is_hierarchical_query_ = other.is_hierarchical_query_; has_prior_ = other.has_prior_; has_reverse_link_ = other.has_reverse_link_; + is_expanded_mview_ = other.is_expanded_mview_; // copy insert into statement if (OB_SUCC(ret) && NULL != other.into_item_) { ObSelectIntoItem *temp_into_item = NULL; @@ -530,6 +532,7 @@ ObSelectStmt::ObSelectStmt() is_hierarchical_query_ = false; has_prior_ = false; has_reverse_link_ = false; + is_expanded_mview_ = false; } ObSelectStmt::~ObSelectStmt() @@ -725,7 +728,8 @@ int ObSelectStmt::do_to_string(char *buf, const int64_t buf_len, int64_t &pos) c K_(is_hierarchical_query), K_(check_option), K_(dblink_id), - K_(is_reverse_link) + K_(is_reverse_link), + K_(is_expanded_mview) ); } } else { @@ -833,13 +837,14 @@ int ObSelectStmt::clear_sharable_expr_reference() } int ObSelectStmt::remove_useless_sharable_expr(ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info) + ObSQLSessionInfo *session_info, + bool explicit_for_col) { int ret = OB_SUCCESS; if (OB_ISNULL(expr_factory)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(ObDMLStmt::remove_useless_sharable_expr(expr_factory, session_info))) { + } else if (OB_FAIL(ObDMLStmt::remove_useless_sharable_expr(expr_factory, session_info, explicit_for_col))) { LOG_WARN("failed to remove useless sharable expr", K(ret)); } else { ObRawExpr *expr = NULL; diff --git a/src/sql/resolver/dml/ob_select_stmt.h b/src/sql/resolver/dml/ob_select_stmt.h index ce9d8ab59..a15385b90 100644 --- a/src/sql/resolver/dml/ob_select_stmt.h +++ b/src/sql/resolver/dml/ob_select_stmt.h @@ -429,6 +429,8 @@ public: bool is_order_siblings() const { return is_order_siblings_; } void set_hierarchical_query(bool is_hierarchical_query) { is_hierarchical_query_ = is_hierarchical_query; } bool is_hierarchical_query() const { return is_hierarchical_query_; } + inline void set_expanded_mview(bool is_expanded_mview) { is_expanded_mview_ = is_expanded_mview; } + inline bool is_expanded_mview() const { return is_expanded_mview_; } int contain_hierarchical_query(bool &contain_hie_query) const; void set_has_prior(bool has_prior) { has_prior_ = has_prior; } bool has_prior() const { return has_prior_; } @@ -454,7 +456,8 @@ public: bool has_hidden_rowid() const; virtual int clear_sharable_expr_reference() override; virtual int remove_useless_sharable_expr(ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info) override; + ObSQLSessionInfo *session_info, + bool explicit_for_col) override; const common::ObIArray& get_search_by_items() const { return search_by_items_; } const common::ObIArray& get_cycle_items() const { return cycle_by_items_; } @@ -707,6 +710,7 @@ private: bool is_hierarchical_query_; bool has_prior_; bool has_reverse_link_; + bool is_expanded_mview_; }; } } diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index ee5d36b77..62d64340e 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -39,6 +39,9 @@ void ObQueryHint::reset() used_trans_hints_.reuse(); qb_name_map_.reuse(); stmt_id_map_.reuse(); + sel_start_id_ = 1; + set_start_id_ = 1; + other_start_id_ = 1; } int ObQueryHint::create_hint_table(ObIAllocator *allocator, ObTableInHint *&table) @@ -313,7 +316,7 @@ int ObQueryHint::init_query_hint(ObIAllocator *allocator, LOG_WARN("failed to create qb name map", K(ret)); } else if (OB_FAIL(reset_duplicate_qb_name())) { LOG_WARN("failed to reset duplicate qb name", K(ret)); - } else if (OB_FAIL(generate_orig_stmt_qb_name(*allocator))) { + } else if (OB_FAIL(generate_orig_stmt_qb_name(*allocator, 0))) { LOG_WARN("failed to generate stmt name after resolve", K(ret)); } else if (OB_FAIL(distribute_hint_to_orig_stmt(stmt))) { LOG_WARN("faild to distribute hint to orig stmt", K(ret)); @@ -387,17 +390,13 @@ int ObQueryHint::adjust_qb_name_for_stmt(ObIAllocator &allocator, return ret; } -int ObQueryHint::generate_orig_stmt_qb_name(ObIAllocator &allocator) +int ObQueryHint::generate_orig_stmt_qb_name(ObIAllocator &allocator, int64_t inited_stmt_count) { int ret = OB_SUCCESS; - int64_t sel_start_id = 1; - int64_t set_start_id = 1; - int64_t other_start_id = 1; char buf[OB_MAX_QB_NAME_LENGTH]; int64_t buf_len = OB_MAX_QB_NAME_LENGTH; ObString qb_name; - qb_name_map_.reuse(); - for (int64_t idx = 0; OB_SUCC(ret) && idx < stmt_id_map_.count(); ++idx) { + for (int64_t idx = inited_stmt_count; OB_SUCC(ret) && idx < stmt_id_map_.count(); ++idx) { QbNames &qb_names = stmt_id_map_.at(idx); const char *stmt_name = get_dml_stmt_name(qb_names.stmt_type_, qb_names.is_set_stmt_); int64_t pos = 0; @@ -415,8 +414,8 @@ int ObQueryHint::generate_orig_stmt_qb_name(ObIAllocator &allocator) LOG_WARN("failed print buf stmt_name", K(ret)); } else { int64_t &id_start = stmt::T_SELECT == qb_names.stmt_type_ - ? (qb_names.is_set_stmt_ ? set_start_id : sel_start_id) - : other_start_id; + ? (qb_names.is_set_stmt_ ? set_start_id_ : sel_start_id_) + : other_start_id_; int64_t old_pos = pos; int64_t cnt = 0; qb_name.reset(); diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index 43f091bf9..0071bbfb4 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -94,7 +94,7 @@ struct ObQueryHint { const ObIArray &src_hash_val, int64_t *sub_num = NULL); - int generate_orig_stmt_qb_name(ObIAllocator &allocator); + int generate_orig_stmt_qb_name(ObIAllocator &allocator, int64_t inited_stmt_count); int generate_qb_name_for_stmt(ObIAllocator &allocator, const ObDMLStmt &stmt, const ObString &src_qb_name, @@ -172,6 +172,9 @@ struct ObQueryHint { ObSEArray used_trans_hints_; ObSEArray stmt_id_map_; // stmt id -> qb name list, position is stmt id hash::ObHashMap qb_name_map_; // qb name -> stmt id + int64_t sel_start_id_; + int64_t set_start_id_; + int64_t other_start_id_; private: DISALLOW_COPY_AND_ASSIGN(ObQueryHint); diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 8f9e9575d..84ba00b2c 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -834,6 +834,7 @@ int ObRawExpr::is_const_inherit_expr(bool &is_const_inherit, || (param_need_replace ? is_not_calculable_expr() : cnt_not_calculable_expr()) || T_FUN_LABEL_SE_SESSION_LABEL == type_ || T_FUN_LABEL_SE_SESSION_ROW_LABEL == type_ + || T_FUN_SYS_LAST_REFRESH_SCN == type_ || (T_FUN_UDF == type_ && !static_cast(this)->is_deterministic()) || T_FUN_SYS_GET_LOCK == type_ @@ -4007,6 +4008,8 @@ bool ObSysFunRawExpr::inner_same_as( T_OP_GET_SYS_VAR == get_expr_type() || (has_flag(IS_STATE_FUNC) && (NULL == check_context || (NULL != check_context && check_context->need_check_deterministic_)))) { + } else if (T_FUN_SYS_LAST_REFRESH_SCN == get_expr_type()) { + bool_ret = get_mview_id() == static_cast(expr).get_mview_id(); } else if (get_expr_class() == expr.get_expr_class()) { //for EXPR_UDF and EXPR_SYS_FUNC const ObSysFunRawExpr *s_expr = static_cast(&expr); @@ -4277,6 +4280,10 @@ int ObSysFunRawExpr::get_name_internal(char *buf, const int64_t buf_len, int64_t } else if (T_FUN_SYS_INNER_ROW_CMP_VALUE == get_expr_type()) { CK(3 == get_param_count()); OZ(get_param_expr(2)->get_name(buf, buf_len, pos, type)); + } else if (T_FUN_SYS_LAST_REFRESH_SCN == get_expr_type()) { + if (OB_FAIL(BUF_PRINTF("%ld", get_mview_id()))) { + LOG_WARN("fail to BUF_PRINTF", K(ret)); + } } else { int64_t i = 0; if (get_param_count() > 1) { diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index 7f66fedbf..f94411bfc 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -3556,6 +3556,8 @@ public: int get_autoinc_nextval_name(char *buf, int64_t buf_len, int64_t &pos) const; void set_op_id(int64_t operator_id) { operator_id_ = operator_id; } int64_t get_op_id() const { return operator_id_; } + void set_mview_id(uint64_t mview_id) { mview_id_ = mview_id; } + uint64_t get_mview_id() const { return mview_id_; } void set_dblink_name(const common::ObString &name) { dblink_name_ = name; } const common::ObString &get_dblink_name() const { return dblink_name_; } void set_dblink_id(int64_t dblink_id) { dblink_id_ = dblink_id; } @@ -3572,7 +3574,8 @@ public: K_(dblink_name), K_(dblink_id), K_(local_session_var), - K_(local_session_var_id)); + K_(local_session_var_id), + K_(mview_id)); virtual int set_local_session_vars(const share::schema::ObLocalSessionVar *local_sys_vars, const ObBasicSessionInfo *session, int64_t ctx_array_idx); @@ -3581,8 +3584,10 @@ private: DISALLOW_COPY_AND_ASSIGN(ObSysFunRawExpr); common::ObString func_name_; common::ObString dblink_name_; - //用于记录rownum表达式归属的count算子的op_id_ - uint64_t operator_id_; + union { + uint64_t operator_id_; // for rownum expr + uint64_t mview_id_; // for last_refresh_scn expr + }; uint64_t dblink_id_; }; diff --git a/src/sql/resolver/expr/ob_raw_expr_copier.cpp b/src/sql/resolver/expr/ob_raw_expr_copier.cpp index 22b200db2..ee943151f 100644 --- a/src/sql/resolver/expr/ob_raw_expr_copier.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_copier.cpp @@ -202,13 +202,23 @@ int ObRawExprCopier::add_expr(const ObRawExpr *from, LOG_WARN("failed to create expr set", K(ret)); } } - if (OB_SUCC(ret)) { - if (OB_FAIL(copied_exprs_.set_refactored(reinterpret_cast(from), - reinterpret_cast(to)))) { + if (OB_FAIL(ret)) { + } else if (OB_SUCCESS != (ret = copied_exprs_.set_refactored(reinterpret_cast(from), + reinterpret_cast(to)))) { + if (OB_UNLIKELY(ret != OB_HASH_EXIST)) { LOG_WARN("faield to add copied expr into map", K(ret)); - } else if (OB_FAIL(new_exprs_.set_refactored(reinterpret_cast(to)))) { - LOG_WARN("failed to add copied expr into set", K(ret)); + } else { + ret = OB_SUCCESS; + uint64_t val = 0; + if (OB_FAIL(copied_exprs_.get_refactored(reinterpret_cast(from), val))) { + LOG_WARN("get expr from hash map failed", K(ret)); + } else if (OB_UNLIKELY(val != reinterpret_cast(to))) { + ret = OB_HASH_EXIST; + LOG_WARN("from expr exists", K(ret), KPC(from), KPC(to), K(val)); + } } + } else if (OB_FAIL(new_exprs_.set_refactored(reinterpret_cast(to)))) { + LOG_WARN("failed to add copied expr into set", K(ret)); } return ret; } diff --git a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp index c2f126aaf..a9fe9c5f6 100644 --- a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp @@ -480,6 +480,7 @@ int ObRawExprInfoExtractor::visit(ObSysFunRawExpr &expr) || (T_FUN_SYS_SYSDATE == expr.get_expr_type() && !lib::is_oracle_mode()) || T_FUN_NORMAL_UDF == expr.get_expr_type() || T_FUN_SYS_GENERATOR == expr.get_expr_type() + || T_FUN_SYS_LAST_REFRESH_SCN == expr.get_expr_type() || (T_FUN_UDF == expr.get_expr_type() && !static_cast(expr).is_deterministic()) || T_FUN_SYS_GET_LOCK == expr.get_expr_type() diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 070d94885..02a247527 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -1195,6 +1195,12 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, ObRawExpr } break; } + case T_FUN_SYS_LAST_REFRESH_SCN: { + if (OB_FAIL(process_last_refresh_scn_node(node, expr))) { + LOG_WARN("failed to process last refresh scn node", K(ret)); + } + break; + } case T_DBLINK_UDF: { if (OB_FAIL(process_dblink_udf_node(node, expr))) { LOG_WARN("failed to process dblink udf node", K(ret), K(node)); @@ -8353,6 +8359,69 @@ int ObRawExprResolverImpl::process_odbc_time_literals(const ObItemType dst_time_ return ret; } +int ObRawExprResolverImpl::process_last_refresh_scn_node(const ParseNode *expr_node, + ObRawExpr *&expr) +{ + int ret = OB_SUCCESS; + expr = NULL; + const ParseNode *child_node = NULL; + ObSysFunRawExpr *func_expr = NULL; + uint64_t mview_id = OB_INVALID_ID; + if (OB_ISNULL(expr_node) || OB_UNLIKELY(1 != expr_node->num_child_) + || OB_ISNULL(child_node = expr_node->children_[0])) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(expr_node), K(child_node)); + } else if (OB_FAIL(ctx_.expr_factory_.create_raw_expr(T_FUN_SYS_LAST_REFRESH_SCN, func_expr))) { + LOG_WARN("fail to create raw expr", K(ret)); + } else if (OB_ISNULL(func_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("func_expr is null"); + } else if (T_INT == child_node->type_) { + mview_id = static_cast(child_node->value_); + } else if (OB_UNLIKELY(T_QUESTIONMARK != child_node->type_) + || OB_ISNULL(ctx_.query_ctx_) || OB_ISNULL(ctx_.param_list_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(get_type_name(child_node->type_)), K(ctx_.query_ctx_), K(ctx_.param_list_)); + } else { // question mark need add const constaint. + ObPCConstParamInfo const_param_info; + const int64_t idx = child_node->value_; + const ObObj &val = ctx_.param_list_->at(idx); + number::ObNumber number; + if (val.is_integer_type()) { + mview_id = static_cast(val.get_int()); + } else if (val.is_decimal_int()) { + bool is_valid = false; + if (OB_FAIL(wide::check_range_valid_uint64(val.get_decimal_int(), val.get_int_bytes(), + is_valid, mview_id))) { + LOG_WARN("check_range_valid_uint64 failed", K(ret), K(val.get_int_bytes())); + } else if (OB_UNLIKELY(!is_valid)) { + ret = OB_OBJ_TYPE_ERROR; + LOG_WARN("failed to get uint64", K(ret), K(val)); + } + } else if (OB_FAIL(val.get_number(number))) { + LOG_WARN("failed to get number", K(ret), K(val.get_meta()), K(val)); + } else if (OB_UNLIKELY(!number.is_valid_uint64(mview_id))) { + ret = OB_OBJ_TYPE_ERROR; + LOG_WARN("failed to get uint64", K(ret), K(number)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(const_param_info.const_idx_.push_back(idx)) + || OB_FAIL(const_param_info.const_params_.push_back(val))) { + LOG_WARN("failed to push back param idx and value", K(ret)); + } else if (OB_FAIL(ctx_.query_ctx_->all_plan_const_param_constraints_.push_back(const_param_info))) { + LOG_WARN("failed to push back const param info", K(ret)); + } + } + + if (OB_SUCC(ret)) { + func_expr->set_mview_id(mview_id); + func_expr->set_func_name(ObString::make_string(N_SYS_LAST_REFRESH_SCN)); + expr = func_expr; + LOG_DEBUG("finish resolve last_refresh_scn expr", K(get_type_name(child_node->type_)), K(func_expr->get_mview_id())); + } + return ret; +} } //namespace sql } //namespace oceanbase diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h index 986de6cef..c1018441d 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h @@ -210,6 +210,7 @@ private: int process_xmlparse_node(const ParseNode *node, ObRawExpr *&expr); void get_special_func_ident_name(ObString &ident_name, const ObItemType func_type); int process_remote_sequence_node(const ParseNode *node, ObRawExpr *&expr); + int process_last_refresh_scn_node(const ParseNode *expr_node, ObRawExpr *&expr); int process_dblink_udf_node(const ParseNode *node, ObRawExpr *&expr); int resolve_dblink_udf_expr(const ParseNode *node, ObQualifiedName &column_ref, diff --git a/src/sql/resolver/mv/ob_mv_checker.cpp b/src/sql/resolver/mv/ob_mv_checker.cpp index 7f571bf11..151435e38 100644 --- a/src/sql/resolver/mv/ob_mv_checker.cpp +++ b/src/sql/resolver/mv/ob_mv_checker.cpp @@ -15,6 +15,7 @@ #include "sql/rewrite/ob_expand_aggregate_utils.h" #include "sql/resolver/mv/ob_mv_checker.h" #include "sql/optimizer/ob_optimizer_util.h" +#include "sql/rewrite/ob_transform_utils.h" namespace oceanbase @@ -24,16 +25,24 @@ namespace sql { int ObMVChecker::check_mv_fast_refresh_valid(const ObSelectStmt *view_stmt, + ObStmtFactory *stmt_factory, ObRawExprFactory *expr_factory, ObSQLSessionInfo *session_info) { int ret = OB_SUCCESS; - if (OB_ISNULL(view_stmt) || OB_ISNULL(expr_factory) || OB_ISNULL(session_info)) { + ObDMLStmt *copied_stmt = NULL; + if (OB_ISNULL(view_stmt) || OB_ISNULL(stmt_factory) + || OB_ISNULL(expr_factory) || OB_ISNULL(session_info)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(view_stmt), K(expr_factory), K(session_info)); + } else if (OB_FAIL(ObTransformUtils::deep_copy_stmt(*stmt_factory, *expr_factory, + view_stmt, copied_stmt))) { + LOG_WARN("failed to deep copy stmt", K(ret)); + } else if (OB_FAIL(copied_stmt->formalize_stmt_expr_reference(expr_factory, session_info, true))) { + LOG_WARN("failed to formalize stmt reference", K(ret)); } else { FastRefreshableNotes notes; - ObMVChecker checker(*view_stmt, *expr_factory, session_info); + ObMVChecker checker(*static_cast(copied_stmt), *expr_factory, session_info); checker.set_fast_refreshable_note(¬es); if (OB_FAIL(checker.check_mv_refresh_type())) { LOG_WARN("failed to check mv refresh type", K(ret)); @@ -125,72 +134,55 @@ int ObMVChecker::check_mv_stmt_refresh_type_basic(const ObSelectStmt &stmt, bool } if (OB_SUCC(ret)) { - bool from_table_valid = false; bool mlog_valid = false; - if (OB_FAIL(check_mv_from_tables(stmt, from_table_valid))) { - LOG_WARN("failed to check mv from tables", K(ret)); - } else if (OB_FAIL(check_mv_dependency_mlog_tables(stmt, mlog_valid))) { + if (OB_FAIL(check_mv_dependency_mlog_tables(stmt, mlog_valid))) { LOG_WARN("failed to check mv table mlog", K(ret)); - } else if (!from_table_valid || !mlog_valid) { + } else if (!mlog_valid) { + is_valid = false; + } + } + + if (OB_SUCC(ret)) { + bool has_dup_exprs = false; + if (OB_FAIL(check_mv_duplicated_exprs(stmt, has_dup_exprs))) { + LOG_WARN("failed to check mv table mlog", K(ret)); + } else if (has_dup_exprs) { is_valid = false; } } return ret; } -int ObMVChecker::check_mv_from_tables(const ObSelectStmt &stmt, bool &is_valid) +int ObMVChecker::check_mv_duplicated_exprs(const ObSelectStmt &stmt, bool &has_dup_exprs) { int ret = OB_SUCCESS; - is_valid = true; - ObSEArray from_tables; - if (stmt.get_from_item_size() == 0) { - is_valid = false; - } else if (OB_FAIL(stmt.get_from_tables(from_tables))) { - LOG_WARN("failed to get from tables", K(ret)); - } else { - int64_t table_item_cnt = 0; - for (int64_t i = 0; is_valid && OB_SUCC(ret) && i < from_tables.count(); ++i) { - if (OB_ISNULL(from_tables.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(i), K(from_tables)); - } else if (from_tables.at(i)->is_basic_table()) { - ++table_item_cnt; - } else if (!from_tables.at(i)->is_joined_table()) { - is_valid = false; - } else { - is_valid = is_mv_joined_tables_valid(from_tables.at(i)); - table_item_cnt += static_cast(from_tables.at(i))->single_table_ids_.count(); - } + has_dup_exprs = false; + ObSEArray tmp_exprs; + const ObIArray &select_items = stmt.get_select_items(); + const ObIArray &group_exprs = stmt.get_group_exprs(); + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + if (has_exist_in_array(tmp_exprs, select_items.at(i).expr_)) { + has_dup_exprs = true; + append_fast_refreshable_note("duplicated select output"); + LOG_WARN("fast refresh not support due to duplicated select output", K(i), K(select_items.at(i))); + } else if (OB_FAIL(tmp_exprs.push_back(select_items.at(i).expr_))) { + LOG_WARN("failed to push back", K(ret)); } - LOG_TRACE("finish check check_mv_from_tables", K(is_valid), K(stmt.get_table_size()), K(table_item_cnt)); - if (OB_SUCC(ret) && is_valid) { - is_valid = stmt.get_table_size() == table_item_cnt; - } - if (OB_SUCC(ret) && !is_valid) { - append_fast_refreshable_note("inline view/join/subquery not support"); + } + + tmp_exprs.reuse(); + for (int64_t i = 0; OB_SUCC(ret) && i < group_exprs.count(); ++i) { + if (has_exist_in_array(tmp_exprs, group_exprs.at(i))) { + has_dup_exprs = true; + append_fast_refreshable_note("duplicated group by expr"); + LOG_WARN("fast refresh not support due to duplicated group by expr", K(i), KPC(group_exprs.at(i))); + } else if (OB_FAIL(tmp_exprs.push_back(group_exprs.at(i)))) { + LOG_WARN("failed to push back", K(ret)); } } return ret; } -bool ObMVChecker::is_mv_joined_tables_valid(const TableItem *table) -{ - int bret = false; - if (OB_ISNULL(table)) { - bret = false; - } else if (table->is_basic_table()) { - bret = true; - } else if (!table->is_joined_table()) { - bret = false; - } else { - const JoinedTable *joined_table = static_cast(table); - bret = joined_table->is_inner_join() - && is_mv_joined_tables_valid(joined_table->left_table_) - && is_mv_joined_tables_valid(joined_table->right_table_); - } - return bret; -} - int ObMVChecker::check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool &is_valid) { int ret = OB_SUCCESS; @@ -201,6 +193,8 @@ int ObMVChecker::check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool || OB_ISNULL(sql_schema_guard = &stmt.get_query_ctx()->sql_schema_guard_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(sql_schema_guard)); + } else if (stmt.get_table_size() == 0) { + is_valid = false; } else { is_valid = true; const ObIArray &tables = stmt.get_table_items(); @@ -212,7 +206,8 @@ int ObMVChecker::check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected table", K(ret), KPC(table)); } else if (OB_UNLIKELY(!table->is_basic_table())) { - /* ObMVChecker::check_mv_from_tables has set refreshable as false, ignore this table here */ + is_valid = false; + append_fast_refreshable_note("basic table allowed only"); } else if (OB_FAIL(sql_schema_guard->get_table_schema(table->ref_id_, table_schema))) { LOG_WARN("failed to get table schema", K(ret)); } else if (OB_ISNULL(table_schema)) { @@ -476,6 +471,9 @@ int ObMVChecker::check_and_expand_mav_aggr(const ObSelectStmt &stmt, expand_aggr_utils.set_expand_for_mv(); if (OB_FAIL(expand_aggr_utils.expand_common_aggr_expr(aggr, replace_expr, all_aggrs))) { LOG_WARN("failed to expand common aggr expr", K(ret)); + } else if (all_aggrs.count() != orig_aggr_count + && OB_FAIL(try_replace_equivalent_count_aggr(stmt, orig_aggr_count, all_aggrs, replace_expr))) { + LOG_WARN("failed to try replace equivalent count aggr ", K(ret)); } else if (all_aggrs.count() != orig_aggr_count) { /* expand aggr generate new aggr, can not fast refresh */ is_valid = false; @@ -502,6 +500,42 @@ int ObMVChecker::check_and_expand_mav_aggr(const ObSelectStmt &stmt, return ret; } +int ObMVChecker::try_replace_equivalent_count_aggr(const ObSelectStmt &stmt, + const int64_t orig_aggr_count, + ObIArray &all_aggrs, + ObRawExpr *&replace_expr) +{ + int ret = OB_SUCCESS; + const ObAggFunRawExpr *aggr = NULL; + const ObAggFunRawExpr *equal_aggr = NULL; + bool aggr_not_support = false; + ObRawExprCopier copier(expr_factory_); + for (int64_t i = orig_aggr_count; !aggr_not_support && OB_SUCC(ret) && i < all_aggrs.count(); ++i) { + if (OB_ISNULL(aggr = all_aggrs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL", K(ret), K(i), K(aggr)); + } else if (T_FUN_COUNT != aggr->get_expr_type() || 1 != aggr->get_real_param_count()) { + aggr_not_support = true; + } else if (OB_FAIL(get_equivalent_count_aggr(stmt, aggr->get_real_param_exprs().at(0), equal_aggr))) { + LOG_WARN("failed to get equivalent count aggr", K(ret)); + } else if (NULL == equal_aggr) { + aggr_not_support = true; + } else if (OB_FAIL(copier.add_replaced_expr(aggr, equal_aggr))) { + LOG_WARN("failed to add replace pair", K(ret)); + } + } + if (OB_SUCC(ret) && !aggr_not_support) { + ObRawExpr *new_replace_expr = NULL; + ObOptimizerUtil::revert_items(all_aggrs, orig_aggr_count); + if (OB_FAIL(copier.copy_on_replace(replace_expr, new_replace_expr))) { + LOG_WARN("failed to generate group by exprs", K(ret)); + } else { + replace_expr = new_replace_expr; + } + } + return ret; +} + bool ObMVChecker::is_basic_aggr(const ObItemType aggr_type) { return T_FUN_COUNT == aggr_type || T_FUN_SUM == aggr_type; @@ -520,20 +554,33 @@ int ObMVChecker::get_dependent_aggr_of_fun_sum(const ObSelectStmt &stmt, || OB_UNLIKELY(T_FUN_SUM != aggr->get_expr_type() || 1 != aggr->get_real_param_count()) || OB_ISNULL(param_expr = aggr->get_real_param_exprs().at(0))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected sum aggr", K(ret), K(aggr)); - } else if (OB_FAIL(get_target_aggr(T_FUN_COUNT, param_expr, stmt.get_aggr_items(), dependent_aggr))) { + LOG_WARN("unexpected sum aggr", K(ret), KPC(aggr)); + } else if (OB_FAIL(get_equivalent_count_aggr(stmt, param_expr, dependent_aggr))) { + LOG_WARN("failed to get equivalent count aggr", K(ret)); + } + return ret; +} + +int ObMVChecker::get_equivalent_count_aggr(const ObSelectStmt &stmt, + const ObRawExpr *param_expr, + const ObAggFunRawExpr *&count_aggr) +{ + int ret = OB_SUCCESS; + count_aggr = NULL; + const ObRawExpr *inner_param_expr = NULL; + if (OB_FAIL(get_target_aggr(T_FUN_COUNT, param_expr, stmt.get_aggr_items(), count_aggr))) { LOG_WARN("failed to check target aggr exist", K(ret)); - } else if (NULL != dependent_aggr && stmt.check_is_select_item_expr(dependent_aggr)) { + } else if (NULL != count_aggr && stmt.check_is_select_item_expr(count_aggr)) { /* do nothing */ } else if (OB_FAIL(get_equivalent_null_check_param(param_expr, inner_param_expr))) { LOG_WARN("failed to get equivalent null check param", K(ret)); } else if (NULL == inner_param_expr) { /* can not get valid dependent aggr */ - dependent_aggr = NULL; - } else if (OB_FAIL(get_target_aggr(T_FUN_COUNT, inner_param_expr, stmt.get_aggr_items(), dependent_aggr))) { + count_aggr = NULL; + } else if (OB_FAIL(get_target_aggr(T_FUN_COUNT, inner_param_expr, stmt.get_aggr_items(), count_aggr))) { LOG_WARN("failed to check target aggr exist", K(ret)); - } else if (NULL != dependent_aggr && !stmt.check_is_select_item_expr(dependent_aggr)) { - dependent_aggr = NULL; + } else if (NULL != count_aggr && !stmt.check_is_select_item_expr(count_aggr)) { + count_aggr = NULL; } return ret; } @@ -596,9 +643,112 @@ int ObMVChecker::get_target_aggr(const ObItemType target_aggr_type, int ObMVChecker::check_mjv_refresh_type(const ObSelectStmt &stmt, ObMVRefreshableType &refresh_type) { int ret = OB_SUCCESS; - UNUSED(stmt); refresh_type = OB_MV_COMPLETE_REFRESH; - append_fast_refreshable_note("mv without group by not support now"); + bool join_type_valid = true; + bool select_valid = true; + if (stmt.get_table_size() <= 1) { + append_fast_refreshable_note("table size not support"); + } else if (stmt.get_table_size() > 5) { + append_fast_refreshable_note("join table size more than 5 not support"); + } else if (OB_FAIL(check_mv_join_type(stmt, join_type_valid))) { + LOG_WARN("failed to check mv join type", K(ret)); + } else if (!join_type_valid) { + append_fast_refreshable_note("outer join not support"); + } else if (OB_FAIL(check_select_contains_all_tables_primary_key(stmt, select_valid))) { + LOG_WARN("failed to check check select contains all tables primary key", K(ret)); + } else if (!select_valid) { + append_fast_refreshable_note("outer join not support"); + } else { + refresh_type = OB_MV_FAST_REFRESH_SIMPLE_MJV; + } + return ret; +} + +int ObMVChecker::check_mv_join_type(const ObSelectStmt &stmt, bool &join_type_valid) +{ + int ret = OB_SUCCESS; + join_type_valid = true; + const ObIArray &joined_tables = stmt.get_joined_tables(); + for (int64_t i = 0; join_type_valid && OB_SUCC(ret) && i < joined_tables.count(); ++i) { + join_type_valid &= is_mv_join_type_valid(joined_tables.at(i)); + } + return ret; +} + +bool ObMVChecker::is_mv_join_type_valid(const TableItem *table) +{ + int bret = false; + if (OB_ISNULL(table)) { + bret = false; + } else if (table->is_basic_table()) { + bret = true; + } else if (!table->is_joined_table()) { + bret = false; + } else { + const JoinedTable *joined_table = static_cast(table); + bret = joined_table->is_inner_join() + && is_mv_join_type_valid(joined_table->left_table_) + && is_mv_join_type_valid(joined_table->right_table_); + } + return bret; +} + +int ObMVChecker::check_select_contains_all_tables_primary_key(const ObSelectStmt &stmt, + bool &contain_all_rowkey) +{ + int ret = OB_SUCCESS; + contain_all_rowkey = false; + if (OB_ISNULL(stmt.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(stmt.get_query_ctx())); + } else { + contain_all_rowkey = true; + int64_t all_rowkey_size = 0; + ObSEArray rowkeys; + ObSqlSchemaGuard &sql_schema_guard = stmt.get_query_ctx()->sql_schema_guard_; + for (int64_t i = 0; contain_all_rowkey && OB_SUCC(ret) && i < stmt.get_table_items().count(); ++i) { + TableItem *table_item = NULL; + const ObTableSchema *table_schema = NULL; + if (OB_ISNULL(table_item = stmt.get_table_items().at(i)) + || OB_UNLIKELY(!table_item->is_basic_table())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table", K(ret), K(i), KPC(table_item)); + } else if (OB_FAIL(sql_schema_guard.get_table_schema(table_item->ref_id_, table_schema))) { + LOG_WARN("table schema not found", K(table_schema)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid table schema", K(ret), K(table_schema)); + } else if (table_schema->is_heap_table()) { + contain_all_rowkey = false; + } else { + all_rowkey_size += table_schema->get_rowkey_info().get_size(); + } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_select_items().count(); ++i) { + const ObRawExpr *expr = NULL; + int64_t idx = OB_INVALID_INDEX; + if (OB_ISNULL(expr = stmt.get_select_items().at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(stmt.get_select_items())); + } else if (!expr->is_column_ref_expr() || !static_cast(expr)->is_rowkey_column()) { + /* do nothing */ + } else if (OB_FAIL(add_var_to_array_no_dup(rowkeys, expr, &idx))) { + LOG_WARN("failed to add_var to array no dup", K(ret)); + } + } + + if (OB_FAIL(ret) || !contain_all_rowkey) { + /* do nothing */ + } else if (all_rowkey_size > rowkeys.count()) { + contain_all_rowkey = false; + } else if (OB_UNLIKELY(rowkeys.count() != all_rowkey_size)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rowkey size", K(ret), K(all_rowkey_size), K(rowkeys.count()), K(rowkeys)); + } else { + contain_all_rowkey = true; + } + } return ret; } diff --git a/src/sql/resolver/mv/ob_mv_checker.h b/src/sql/resolver/mv/ob_mv_checker.h index 88eba85d5..c516e0f32 100644 --- a/src/sql/resolver/mv/ob_mv_checker.h +++ b/src/sql/resolver/mv/ob_mv_checker.h @@ -28,6 +28,7 @@ enum ObMVRefreshableType OB_MV_REFRESH_INVALID = 0, // refresh type not set OB_MV_COMPLETE_REFRESH, // can not fast refresh OB_MV_FAST_REFRESH_SIMPLE_MAV, // fast refresh for single table MAV + OB_MV_FAST_REFRESH_SIMPLE_MJV, // fast refresh for inner join MJV }; struct FastRefreshableNotes @@ -62,21 +63,25 @@ class ObMVChecker ~ObMVChecker() {} static int check_mv_fast_refresh_valid(const ObSelectStmt *view_stmt, + ObStmtFactory *stmt_factory, ObRawExprFactory *expr_factory, ObSQLSessionInfo *session_info); int check_mv_refresh_type(); ObMVRefreshableType get_refersh_type() const { return refresh_type_; }; static bool is_basic_aggr(const ObItemType aggr_type); static int get_dependent_aggr_of_fun_sum(const ObSelectStmt &stmt, const ObAggFunRawExpr *aggr, const ObAggFunRawExpr *&dependent_aggr); + static int get_equivalent_count_aggr(const ObSelectStmt &stmt, const ObRawExpr *param_expr, const ObAggFunRawExpr *&count_aggr); const ObSelectStmt &get_stmt() const { return stmt_; } const ObIArray> &get_expand_aggrs() const { return expand_aggrs_; } int get_mlog_table_schema(const TableItem *table, const share::schema::ObTableSchema *&mlog_schema) const; void set_fast_refreshable_note(FastRefreshableNotes *note) { fast_refreshable_note_ = note; } private: int check_mv_stmt_refresh_type_basic(const ObSelectStmt &stmt, bool &is_valid); - int check_mv_from_tables(const ObSelectStmt &stmt, bool &is_valid); - bool is_mv_joined_tables_valid(const TableItem *table); + int check_mv_join_type(const ObSelectStmt &stmt, bool &join_type_valid); + bool is_mv_join_type_valid(const TableItem *table); + int check_select_contains_all_tables_primary_key(const ObSelectStmt &stmt, bool &contain_all_rowkey); int check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool &is_valid); + int check_mv_duplicated_exprs(const ObSelectStmt &stmt, bool &has_dup_exprs); bool check_mlog_table_valid(const share::schema::ObTableSchema *table_schema, const ObIArray &columns, const share::schema::ObTableSchema &mlog_schema, @@ -95,6 +100,10 @@ private: ObIArray &all_aggrs, ObIArray> &expand_aggrs, bool &is_valid); + int try_replace_equivalent_count_aggr(const ObSelectStmt &stmt, + const int64_t orig_aggr_count, + ObIArray &all_aggrs, + ObRawExpr *&replace_expr); static int get_equivalent_null_check_param(const ObRawExpr *param_expr, const ObRawExpr *&inner_param_expr); static int get_target_aggr(const ObItemType target_aggr_type, const ObRawExpr *param_expr, @@ -111,6 +120,7 @@ private: ObRawExprFactory &expr_factory_; ObSQLSessionInfo *session_info_; FastRefreshableNotes *fast_refreshable_note_; + DISALLOW_COPY_AND_ASSIGN(ObMVChecker); }; } diff --git a/src/sql/resolver/mv/ob_mv_dep_utils.cpp b/src/sql/resolver/mv/ob_mv_dep_utils.cpp new file mode 100644 index 000000000..e8afdddb4 --- /dev/null +++ b/src/sql/resolver/mv/ob_mv_dep_utils.cpp @@ -0,0 +1,252 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_RESV +#include "sql/resolver/mv/ob_mv_dep_utils.h" +#include "lib/mysqlclient/ob_mysql_transaction.h" +#include "lib/string/ob_sql_string.h" +#include "share/schema/ob_schema_utils.h" +#include "share/ob_dml_sql_splicer.h" +#include "share/inner_table/ob_inner_table_schema_constants.h" +#include "share/schema/ob_dependency_info.h" + +namespace oceanbase +{ +using namespace common; +using namespace share; +using namespace share::schema; +using namespace common::sqlclient; +namespace sql +{ +bool ObMVDepInfo::is_valid() const +{ + return (OB_INVALID_TENANT_ID != tenant_id_) + && (OB_INVALID_ID != mview_id_) + && (OB_INVALID_ID != p_obj_); +} + +int ObMVDepUtils::get_mview_dep_infos( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + ObIArray &dep_infos) +{ + int ret = OB_SUCCESS; + if ((OB_INVALID_TENANT_ID == tenant_id) + || (OB_INVALID_ID == mview_table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or mview_table_id", KR(ret), K(tenant_id), K(mview_table_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString sql; + ObMySQLResult *result = NULL; + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + if (OB_FAIL(sql.assign_fmt("SELECT p_order, p_obj, p_type, qbcid, flags FROM %s" + " WHERE tenant_id = %lu AND mview_id = %lu ORDER BY p_order", + OB_ALL_MVIEW_DEP_TNAME, + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), + mview_table_id))) { + LOG_WARN("failed to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("failed to execute read", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret), KP(result)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else { + ObMVDepInfo dep_info; + EXTRACT_INT_FIELD_MYSQL(*result, "p_order", dep_info.p_order_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "p_obj", dep_info.p_obj_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "p_type", dep_info.p_type_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "qbcid", dep_info.qbcid_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "flags", dep_info.flags_, int64_t); + + if (OB_SUCC(ret)) { + dep_info.tenant_id_ = tenant_id; + dep_info.mview_id_ = mview_table_id; + if (OB_FAIL(dep_infos.push_back(dep_info))) { + LOG_WARN("failed to add dep info", KR(ret), K(dep_info)); + } + } + } + } + } + } + } + return ret; +} + +int ObMVDepUtils::insert_mview_dep_infos( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + const ObIArray &dep_infos) +{ + int ret = OB_SUCCESS; + if ((OB_INVALID_TENANT_ID == tenant_id) + || (OB_INVALID_ID == mview_table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or mview_table_id", KR(ret), K(tenant_id), K(mview_table_id)); + } else { + ObDMLSqlSplicer dml; + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + for (int64_t i = 0; OB_SUCC(ret) && (i < dep_infos.count()); ++i) { + const ObMVDepInfo &dep_info = dep_infos.at(i); + if (!dep_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid dep info", KR(ret), K(dep_info)); + } else if (OB_FAIL(dml.add_pk_column("tenant_id", + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id))) + || OB_FAIL(dml.add_pk_column("mview_id", mview_table_id)) + || OB_FAIL(dml.add_column("p_order", dep_info.p_order_)) + || OB_FAIL(dml.add_column("p_obj", dep_info.p_obj_)) + || OB_FAIL(dml.add_column("p_type", dep_info.p_type_)) + || OB_FAIL(dml.add_column("qbcid", dep_info.qbcid_)) + || OB_FAIL(dml.add_column("flags", dep_info.flags_))) { + LOG_WARN("failed to add column", KR(ret), K(dep_info)); + } else if (OB_FAIL(dml.finish_row())) { + LOG_WARN("failed to finish dml row", KR(ret)); + } + } + if (OB_SUCC(ret)) { + int64_t affected_rows = 0; + ObSqlString sql; + if (OB_FAIL(dml.splice_batch_insert_sql(OB_ALL_MVIEW_DEP_TNAME, sql))) { + LOG_WARN("failed to splice batch insert sql", KR(ret)); + } else if (OB_FAIL(sql_client.write(exec_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("failed to execute write", KR(ret)); + } else if (affected_rows != dep_infos.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected rows does not match the count of dep infos", + KR(ret), K(affected_rows), K(dep_infos.count())); + } + } + } + + return ret; +} + +int ObMVDepUtils::delete_mview_dep_infos( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id) +{ + int ret = OB_SUCCESS; + if ((OB_INVALID_TENANT_ID == tenant_id) + || (OB_INVALID_ID == mview_table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or mview_table_id", KR(ret), K(tenant_id), K(mview_table_id)); + } else { + ObSqlString sql; + int64_t affected_rows = 0; + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE tenant_id = %ld AND mview_id = %ld", + OB_ALL_MVIEW_DEP_TNAME, + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), + mview_table_id))) { + LOG_WARN("failed to delete from __all_mview_dep table", + KR(ret), K(tenant_id), K(mview_table_id)); + } else if (OB_FAIL(sql_client.write(exec_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("failed to execute write", KR(ret), K(sql)); + } + } + + return ret; +} + +int ObMVDepUtils::convert_to_mview_dep_infos( + const ObIArray &deps, + ObIArray &mv_deps) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && (i < deps.count()); ++i) { + const ObDependencyInfo &dep_info = deps.at(i); + ObMVDepInfo mv_dep; + mv_dep.tenant_id_ = dep_info.get_tenant_id(); + mv_dep.mview_id_ = dep_info.get_dep_obj_id(); + mv_dep.p_order_ = dep_info.get_order(); + mv_dep.p_obj_ = dep_info.get_ref_obj_id(); + mv_dep.p_type_ = static_cast(dep_info.get_ref_obj_type()); + if (OB_FAIL(mv_deps.push_back(mv_dep))) { + LOG_WARN("failed to add mv dep to array", KR(ret), K(mv_dep)); + } + } + + return ret; +} + +int ObMVDepUtils::get_table_ids_only_referenced_by_given_mv( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + ObIArray &ref_table_ids) +{ + int ret = OB_SUCCESS; + if ((OB_INVALID_TENANT_ID == tenant_id) + || (OB_INVALID_ID == mview_table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or mview_table_id", + KR(ret), K(tenant_id), K(mview_table_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString sql; + ObMySQLResult *result = NULL; + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + if (OB_FAIL(sql.assign_fmt("select a.p_obj from" + " (select p_obj, count(*) cnt from %s group by p_obj) a," + " (select p_obj, count(*) cnt from %s where tenant_id = %lu" + " and mview_id = %lu group by p_obj) b" + " where a.p_obj = b.p_obj and a.cnt = b.cnt", + OB_ALL_MVIEW_DEP_TNAME, + OB_ALL_MVIEW_DEP_TNAME, + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), + mview_table_id))) { + LOG_WARN("failed to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("failed to execute read", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret), KP(result)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else { + uint64_t ref_table_id = OB_INVALID_ID; + EXTRACT_INT_FIELD_MYSQL(*result, "p_obj", ref_table_id, uint64_t); + if (OB_SUCC(ret)) { + if (OB_FAIL(ref_table_ids.push_back(ref_table_id))) { + LOG_WARN("failed to add ref table id to array", KR(ret), K(ref_table_id)); + } + } + } + } + } + } + } + return ret; +} +} // end of sql +} // end of oceanbase \ No newline at end of file diff --git a/src/sql/resolver/mv/ob_mv_dep_utils.h b/src/sql/resolver/mv/ob_mv_dep_utils.h new file mode 100644 index 000000000..9963a22ac --- /dev/null +++ b/src/sql/resolver/mv/ob_mv_dep_utils.h @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_RESOLVER_MV_OB_MV_DEP_UTILS_H_ +#define OCEANBASE_SQL_RESOLVER_MV_OB_MV_DEP_UTILS_H_ + +#include "lib/ob_define.h" +#include "lib/container/ob_array.h" +#include "lib/utility/ob_print_utils.h" +#include "lib/mysqlclient/ob_isql_client.h" + +namespace oceanbase +{ +namespace share +{ +namespace schema +{ +class ObDependencyInfo; +} +} +namespace sql +{ +class ObMVDepInfo +{ +public: + ObMVDepInfo() : tenant_id_(OB_INVALID_TENANT_ID), + mview_id_(OB_INVALID_ID), + p_order_(0), + p_obj_(OB_INVALID_ID), + p_type_(0), + qbcid_(0), + flags_(0) {} + virtual ~ObMVDepInfo() {} + bool is_valid() const; + + TO_STRING_KV(K_(tenant_id), K_(mview_id), K_(p_order), K_(p_obj), K_(p_type), K_(qbcid), K_(flags)); + +public: + uint64_t tenant_id_; + uint64_t mview_id_; + int64_t p_order_; + uint64_t p_obj_; + int64_t p_type_; + int64_t qbcid_; + int64_t flags_; +}; + +class ObMVDepUtils +{ +public: + static int get_mview_dep_infos(common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + common::ObIArray &dep_infos); + static int insert_mview_dep_infos(common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + const common::ObIArray &dep_infos); + static int delete_mview_dep_infos(common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id); + static int convert_to_mview_dep_infos( + const common::ObIArray &deps, + common::ObIArray &mv_deps); + static int get_table_ids_only_referenced_by_given_mv( + common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + common::ObIArray &ref_table_ids); +}; +} // end of sql +} // end of oceanbase +#endif \ No newline at end of file diff --git a/src/sql/resolver/mv/ob_mv_printer.cpp b/src/sql/resolver/mv/ob_mv_printer.cpp index 2ffc2d3e9..2786e14df 100644 --- a/src/sql/resolver/mv/ob_mv_printer.cpp +++ b/src/sql/resolver/mv/ob_mv_printer.cpp @@ -26,6 +26,7 @@ namespace sql const ObString ObMVPrinter::DELTA_TABLE_VIEW_NAME = "DLT_T$$"; const ObString ObMVPrinter::DELTA_BASIC_MAV_VIEW_NAME = "DLT_BASIC_MV$$"; const ObString ObMVPrinter::DELTA_MAV_VIEW_NAME = "DLT_MV$$"; +const ObString ObMVPrinter::INNER_RT_MV_VIEW_NAME = "INNER_RT_MV$$"; // column name const ObString ObMVPrinter::HEAP_TABLE_ROWKEY_COL_NAME = "M_ROW$$"; const ObString ObMVPrinter::OLD_NEW_COL_NAME = "OLD_NEW$$"; @@ -74,7 +75,7 @@ int ObMVPrinter::print_mv_operators(const share::schema::ObTableSchema &mv_schem ObObjPrintParams obj_print_params(view_stmt.get_query_ctx()->get_timezone_info()); obj_print_params.print_origin_stmt_ = true; for (int64_t i = 0; OB_SUCC(ret) && i < dml_stmts.count(); ++i) { - if (OB_FAIL(ObSQLUtils::reconstruct_sql(str_alloc, dml_stmts.at(i), operators.at(i), schema_guard))) { + if (OB_FAIL(ObSQLUtils::reconstruct_sql(str_alloc, dml_stmts.at(i), operators.at(i), schema_guard, obj_print_params))) { LOG_WARN("fail to reconstruct sql", K(ret)); } else { LOG_TRACE("generate one mv operator", K(i), K(operators.at(i))); @@ -88,47 +89,117 @@ int ObMVPrinter::gen_mv_operator_stmts(ObIArray &dml_stmts) { int ret = OB_SUCCESS; dml_stmts.reuse(); + ObSelectStmt *sel_stmt = NULL; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("printer is not inited", K(ret)); - } else { - switch (mv_checker_.get_refersh_type()) { - case OB_MV_FAST_REFRESH_SIMPLE_MAV: { - if (OB_FAIL(gen_refresh_dmls_for_simple_mav(dml_stmts))) { - LOG_WARN("fail to gen refresh dmls for simple mav", K(ret)); - } - break; - } - default: { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected refresh type", K(ret), K(mv_checker_.get_refersh_type())); - break; - } + } else if (!for_rt_expand_) { + if (OB_FAIL(gen_refresh_dmls_for_mv(dml_stmts))) { + LOG_WARN("failed to gen refresh dmls for mv", K(ret)); } - } - return ret; -} - -int ObMVPrinter::gen_refresh_dmls_for_simple_mav(ObIArray &dml_stmts) -{ - int ret = OB_SUCCESS; - dml_stmts.reuse(); - ObMergeStmt *merge_stmt = NULL; - if (for_rt_expand_) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not supported now", K(ret)); - } else if (lib::is_mysql_mode()) { - if (OB_FAIL(gen_update_insert_delete_for_simple_mav(dml_stmts))) { - LOG_WARN("failed to gen update insert delete for simple mav", K(ret)); - } - } else if (OB_FAIL(gen_merge_for_simple_mav(merge_stmt))) { - LOG_WARN("failed to gen merge into for simple mav", K(ret)); - } else if (OB_FAIL(dml_stmts.push_back(merge_stmt))) { + } else if (OB_FAIL(gen_real_time_view_for_mv(sel_stmt))) { + LOG_WARN("failed to gen real time view for mv", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(sel_stmt))) { LOG_WARN("failed to push back", K(ret)); } return ret; } +int ObMVPrinter::gen_refresh_dmls_for_mv(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + dml_stmts.reuse(); + switch (mv_checker_.get_refersh_type()) { + case OB_MV_FAST_REFRESH_SIMPLE_MAV: { + ObMergeStmt *merge_stmt = NULL; + if (lib::is_mysql_mode()) { + if (OB_FAIL(gen_update_insert_delete_for_simple_mav(dml_stmts))) { + LOG_WARN("failed to gen update insert delete for simple mav", K(ret)); + } + } else if (OB_FAIL(gen_merge_for_simple_mav(merge_stmt))) { + LOG_WARN("failed to gen merge into for simple mav", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(merge_stmt))) { + LOG_WARN("failed to push back", K(ret)); + } + break; + } + case OB_MV_FAST_REFRESH_SIMPLE_MJV: { + if (OB_FAIL(gen_delete_insert_for_simple_mjv(dml_stmts))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to gen refresh dmls for simple mjv", K(ret)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected refresh type", K(ret), K(mv_checker_.get_refersh_type())); + break; + } + } + return ret; +} + +int ObMVPrinter::gen_real_time_view_for_mv(ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + switch (mv_checker_.get_refersh_type()) { + case OB_MV_FAST_REFRESH_SIMPLE_MAV: { + if (OB_FAIL(gen_real_time_view_for_simple_mav(sel_stmt))) { + LOG_WARN("failed to gen real time view for simple mav", K(ret)); + } + break; + } + case OB_MV_FAST_REFRESH_SIMPLE_MJV: { + if (OB_FAIL(gen_real_time_view_for_simple_mjv(sel_stmt))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to gen real time view for simple mjv", K(ret)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected mv type for real-time mview", K(ret), K(mv_checker_.get_refersh_type())); + break; + } + } + return ret; +} + +int ObMVPrinter::gen_delete_insert_for_simple_mjv(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + dml_stmts.reuse(); + if (OB_FAIL(gen_delete_for_simple_mjv(dml_stmts))) { + LOG_WARN("failed to gen delete operators for simple mjv", K(ret)); + } else if (OB_FAIL(gen_insert_into_select_for_simple_mjv(dml_stmts))) { + LOG_WARN("failed to gen delete operators for simple mjv", K(ret)); + } + return ret; +} + +int ObMVPrinter::gen_real_time_view_for_simple_mjv(ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt = NULL; + ObSelectStmt *access_mv_stmt = NULL; + ObSEArray access_delta_stmts; + if (OB_FAIL(create_simple_stmt(sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(gen_access_mv_data_for_simple_mjv(access_mv_stmt))) { + LOG_WARN("failed to generate ", K(ret)); + } else if (OB_FAIL(gen_access_delta_data_for_simple_mjv(access_delta_stmts))) { + LOG_WARN("failed to generate ", K(ret)); + } else if (OB_FAIL(sel_stmt->get_set_query().push_back(access_mv_stmt) + || OB_FAIL(append(sel_stmt->get_set_query(), access_delta_stmts)))) { + LOG_WARN("failed to set set query", K(ret)); + } else { + sel_stmt->assign_set_all(); + sel_stmt->assign_set_op(ObSelectStmt::UNION); + } + return ret; +} + +// generate update/insert/delete dml to fast refresh simple mav in mysql mode int ObMVPrinter::gen_update_insert_delete_for_simple_mav(ObIArray &dml_stmts) { int ret = OB_SUCCESS; @@ -376,6 +447,223 @@ int ObMVPrinter::gen_delete_conds(const ObIArray &mv_column return ret; } +/* + select inner_rt_mv.*, sum_c3/cnt_c3 as avg_c3 + from (select nvl(mv.c1, d_mv.c1) as c1, + nvl(mv.c2, d_mv.c2) as c2, + (case when mv.cnt is null then d_mv.cnt, + when d_mv.cnt is null then mv.cnt, + else mv.cnt + d_mv.cnt) as cnt + (case when mv.cnt_c3 is null then d_mv.cnt_c3, + when d_mv.cnt_c3 is null then mv.cnt_c3, + else mv.cnt_c3 + d_mv.cnt) as cnt_c3 + (case when mv.sum_c3 is null then d_mv.sum_c3, + when d_mv.sum_c3 is null then mv.sum_c3, + else mv.sum_c3 + d_mv.sum_c3) as sum_c3 + from mv full join d_mv on mv.c1 <=> d_mv.c1 and mv.c2 <=> d_mv.c2) inner_rt_mv + where cnt > 0; +*/ +int ObMVPrinter::gen_real_time_view_for_simple_mav(ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt = NULL; + TableItem *view_table = NULL; + if (OB_FAIL(create_simple_stmt(sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(gen_inner_real_time_view_for_mav(*sel_stmt, view_table))) { + LOG_WARN("failed to generate real time view filter", K(ret)); + } else if (OB_FAIL(gen_select_items_for_mav(view_table->get_table_name(), + view_table->table_id_, + sel_stmt->get_select_items()))) { + LOG_WARN("failed to generate select items for mav", K(ret)); + } else if (!mv_checker_.get_stmt().is_scala_group_by() + && OB_FAIL(gen_real_time_view_filter_for_mav(*sel_stmt))) { + LOG_WARN("failed to generate real time view filter", K(ret)); + } + return ret; +} + +int ObMVPrinter::gen_real_time_view_filter_for_mav(ObSelectStmt &sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt.get_condition_exprs().reuse(); + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + ObIArray &select_items = sel_stmt.get_select_items(); + int64_t idx = OB_INVALID_INDEX; + ObRawExpr *expr = NULL; + ObRawExpr *filter_expr = NULL; + for (int64_t i = 0; OB_INVALID_INDEX == idx && OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + if (OB_ISNULL(expr = orig_select_items.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(expr)); + } else if (T_FUN_COUNT == expr->get_expr_type() + && static_cast(expr)->get_real_param_exprs().empty()) { + idx = i; + } + } + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(idx < 0 || idx >= select_items.count() + || orig_select_items.count() != select_items.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx or select item count", K(ret), K(idx), K(orig_select_items.count()), K(select_items.count())); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_GT, select_items.at(idx).expr_, exprs_.int_zero_, filter_expr))) { + LOG_WARN("failed to build greater op expr", K(ret)); + } else if (OB_FAIL(sel_stmt.get_condition_exprs().push_back(filter_expr))) { + LOG_WARN("failed to push back where conds", K(ret)); + } + return ret; +} + +int ObMVPrinter::gen_inner_real_time_view_for_mav(ObSelectStmt &sel_stmt, TableItem *&view_table) +{ + int ret = OB_SUCCESS; + ObSelectStmt *view_stmt = NULL; + if (OB_FAIL(create_simple_stmt(view_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(gen_inner_real_time_view_tables_for_mav(*view_stmt))) { + LOG_WARN("failed to generate real_time_view tables for mav", K(ret)); + } else if (OB_FAIL(gen_inner_real_time_view_select_list_for_mav(*view_stmt))) { + LOG_WARN("failed to generate select list", K(ret)); + } else if (OB_FAIL(create_simple_table_item(&sel_stmt, INNER_RT_MV_VIEW_NAME, view_table, view_stmt))) { + LOG_WARN("failed to create simple table item", K(ret)); + } + return ret; +} + +int ObMVPrinter::gen_inner_real_time_view_tables_for_mav(ObSelectStmt &sel_stmt) +{ + int ret = OB_SUCCESS; + ObSelectStmt *delta_view_stmt = NULL; + TableItem *mv_table = NULL; + TableItem *delta_mv_table = NULL; + JoinedTable *joined_table = NULL; + void *ptr = NULL; + if (OB_FAIL(create_simple_table_item(&sel_stmt, mv_schema_.get_table_name(), mv_table, NULL, false))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(gen_simple_mav_delta_mv_view(delta_view_stmt))) { // for complex mav, just replace this function + LOG_WARN("failed gen simple source stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(&sel_stmt, DELTA_BASIC_MAV_VIEW_NAME, delta_mv_table, delta_view_stmt, false))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_ISNULL(ptr = (alloc_.alloc(sizeof(JoinedTable))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret)); + } else { + joined_table = new (ptr) JoinedTable(); + joined_table->type_ = TableItem::JOINED_TABLE; + joined_table->table_id_ = sel_stmt.get_query_ctx()->available_tb_id_--; + joined_table->joined_type_ = ObJoinType::FULL_OUTER_JOIN; + joined_table->left_table_ = delta_mv_table; + joined_table->right_table_ = mv_table; + mv_table->database_name_ = mv_db_name_; + if (OB_FAIL(sel_stmt.add_from_item(joined_table->table_id_, true))) { + LOG_WARN("failed to add from item", K(ret)); + } else if (OB_FAIL(sel_stmt.add_joined_table(joined_table))) { + LOG_WARN("failed to add joined table into stmt", K(ret)); + } else if (OB_FAIL(joined_table->single_table_ids_.push_back(delta_mv_table->table_id_)) + || OB_FAIL(joined_table->single_table_ids_.push_back(mv_table->table_id_))) { + LOG_WARN("push back single table id failed", K(ret)); + } else { + const ObIArray &group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); + const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); + ObRawExpr *col1 = NULL; + ObRawExpr *col2 = NULL; + ObRawExpr *match_cond = NULL; + // add on condition + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + const SelectItem &select_item = select_items.at(i); + if (OB_ISNULL(select_item.expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected select item", K(ret), K(i), K(select_items)); + } else if (!ObOptimizerUtil::find_item(group_by_exprs, select_item.expr_)) { + /* do nothing */ + } else if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), select_item.alias_name_, mv_table->table_id_, col1)) + || OB_FAIL(create_simple_column_expr(delta_mv_table->get_table_name(), select_item.alias_name_, delta_mv_table->table_id_, col2)) + || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_NSEQ, col1, col2, match_cond))) { + LOG_WARN("failed to build null safe equal expr", K(ret)); + } else if (OB_FAIL(joined_table->join_conditions_.push_back(match_cond))) { + LOG_WARN("failed to push back join conditions", K(ret)); + } + } + } + } + return ret; +} + +int ObMVPrinter::gen_inner_real_time_view_select_list_for_mav(ObSelectStmt &sel_stmt) +{ + int ret = OB_SUCCESS; + ObIArray &select_items = sel_stmt.get_select_items(); + select_items.reuse(); + const ObIArray &orig_group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); + const ObIArray &orig_aggr_exprs = mv_checker_.get_stmt().get_aggr_items(); + TableItem *table1 = NULL; + TableItem *table2 = NULL; + ObRawExpr *col1 = NULL; + ObRawExpr *col2 = NULL; + ObRawExpr *is_null = NULL; + ObCaseOpRawExpr *case_when_expr = NULL; + const ObRawExpr *const_expr = NULL; + ObOpRawExpr *add_expr = NULL; + if (OB_UNLIKELY(2 != sel_stmt.get_table_size()) + || OB_ISNULL(table1 = sel_stmt.get_table_item(0)) + || OB_ISNULL(table2 = sel_stmt.get_table_item(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected inner real time view for mav", K(ret), K(sel_stmt.get_table_size()), K(table1), K(table2)); + } + + // add select list for group by + for (int64_t i = 0; OB_SUCC(ret) && i < orig_group_by_exprs.count(); ++i) { + SelectItem sel_item; + sel_item.is_real_alias_ = true; + if (OB_FAIL(get_mv_select_item_name(orig_group_by_exprs.at(i), sel_item.alias_name_))) { + LOG_WARN("failed to get mv select item name", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(table1->get_table_name(), sel_item.alias_name_, table1->table_id_, col1)) + || OB_FAIL(create_simple_column_expr(table2->get_table_name(), sel_item.alias_name_, table2->table_id_, col2))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(add_nvl_above_exprs(col1, col2, sel_item.expr_))) { + LOG_WARN("failed to add nvl above exprs", K(ret)); + } else if (OB_FAIL(select_items.push_back(sel_item))) { + LOG_WARN("failed to pushback", K(ret)); + } + } + + // add select list for basic aggr + for (int64_t i = 0; OB_SUCC(ret) && i < orig_aggr_exprs.count(); ++i) { + SelectItem sel_item; + sel_item.is_real_alias_ = true; + if (OB_ISNULL(const_expr = orig_aggr_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(const_expr)); + } else if (!ObMVChecker::is_basic_aggr(const_expr->get_expr_type())) { + /* do nothing */ + } else if (OB_FAIL(get_mv_select_item_name(const_expr, sel_item.alias_name_))) { + LOG_WARN("failed to get mv select item name", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(table1->get_table_name(), sel_item.alias_name_, table1->table_id_, col1)) + || OB_FAIL(create_simple_column_expr(table2->get_table_name(), sel_item.alias_name_, table2->table_id_, col2))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(expr_factory_.create_raw_expr(T_OP_CASE, case_when_expr))) { + LOG_WARN("create add expr failed", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_is_not_null_expr(expr_factory_, col1, false, is_null)) + || OB_FAIL(case_when_expr->add_when_param_expr(is_null)) + || OB_FAIL(case_when_expr->add_then_param_expr(col2))) { + LOG_WARN("failed to build and add is null / then exprs", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_is_not_null_expr(expr_factory_, col2, false, is_null)) + || OB_FAIL(case_when_expr->add_when_param_expr(is_null)) + || OB_FAIL(case_when_expr->add_then_param_expr(col1))) { + LOG_WARN("failed to build and add is null / then exprs", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_add_expr(expr_factory_, col1, col2, add_expr))) { + LOG_WARN("failed to build add expr", K(ret)); + } else { + case_when_expr->set_default_param_expr(add_expr); + sel_item.expr_ = case_when_expr; + if (OB_FAIL(select_items.push_back(sel_item))) { + LOG_WARN("failed to pushback", K(ret)); + } + } + } + return ret; +} + /* merge into mv using delta_mv d_mv @@ -403,6 +691,7 @@ int ObMVPrinter::gen_delete_conds(const ObIArray &mv_column ... where d_mv.d_cnt <> 0; */ +// generate merge into to fast refresh simple mav in oracle mode int ObMVPrinter::gen_merge_for_simple_mav(ObMergeStmt *&merge_stmt) { int ret = OB_SUCCESS; @@ -443,9 +732,6 @@ int ObMVPrinter::gen_merge_tables(ObMergeStmt &merge_stmt, LOG_WARN("failed gen simple source stmt", K(ret)); } else if (OB_FAIL(create_simple_table_item(&merge_stmt, DELTA_BASIC_MAV_VIEW_NAME, source_table, source_stmt, false))) { LOG_WARN("failed to create simple table item", K(ret)); - } else if (OB_ISNULL(target_table) || OB_ISNULL(source_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(target_table), K(source_table)); } else { target_table->database_name_ = mv_db_name_; merge_stmt.set_target_table_id(target_table->table_id_); @@ -454,10 +740,6 @@ int ObMVPrinter::gen_merge_tables(ObMergeStmt &merge_stmt, return ret; } -// generate insert values in 3 steps: -// 1. group by exprs and count aggr -// 2. sum aggr -// 3. other select exprs int ObMVPrinter::gen_insert_values_and_desc(const TableItem *target_table, const TableItem *source_table, ObIArray &target_columns, @@ -466,59 +748,90 @@ int ObMVPrinter::gen_insert_values_and_desc(const TableItem *target_table, int ret = OB_SUCCESS; target_columns.reuse(); values_exprs.reuse(); - const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); + typedef ObSEArray SelectItemArray; + SMART_VAR(SelectItemArray, select_items) { + if (OB_ISNULL(target_table) || OB_ISNULL(source_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("target table or source table is null", K(ret), K(target_table), K(source_table)); + } else if (OB_FAIL(gen_select_items_for_mav(source_table->get_table_name(), + source_table->table_id_, + select_items))) { + } else if (OB_FAIL(target_columns.prepare_allocate(select_items.count())) + || OB_FAIL(values_exprs.prepare_allocate(select_items.count()))) { + LOG_WARN("failed to prepare allocate arrays", K(ret)); + } else { + ObRawExpr *target_col = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + if (OB_FAIL(create_simple_column_expr(target_table->get_table_name(), select_items.at(i).alias_name_, + target_table->table_id_, target_col))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else { + target_columns.at(i) = static_cast(target_col); + values_exprs.at(i) = select_items.at(i).expr_; + } + } + } + } + return ret; +} + +// generate select items in 3 steps: +// 1. group by exprs and count aggr +// 2. sum aggr +// 3. other select exprs +int ObMVPrinter::gen_select_items_for_mav(const ObString &table_name, + const uint64_t table_id, + ObIArray &select_items) +{ + int ret = OB_SUCCESS; + select_items.reuse(); + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); const ObIArray &orig_group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); + ObRawExpr *orig_expr = NULL; ObRawExpr *expr = NULL; - ObRawExpr *target_expr = NULL; - ObRawExpr *source_expr = NULL; ObRawExprCopier copier(expr_factory_); - if (OB_ISNULL(target_table) || OB_ISNULL(source_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("target table or source table is null", K(ret), K(target_table), K(source_table)); - } else if (OB_FAIL(target_columns.prepare_allocate(select_items.count())) - || OB_FAIL(values_exprs.prepare_allocate(select_items.count()))) { + if (OB_FAIL(select_items.prepare_allocate(orig_select_items.count()))) { LOG_WARN("failed to prepare allocate arrays", K(ret)); } // 1. add group by exprs and count aggr - for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { - const SelectItem &select_item = select_items.at(i); - if (OB_ISNULL(expr = select_item.expr_)) { + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + SelectItem &select_item = select_items.at(i); + select_item.is_real_alias_ = true; + select_item.alias_name_ = orig_select_items.at(i).alias_name_; + if (OB_ISNULL(orig_expr = orig_select_items.at(i).expr_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected select item", K(ret), K(i), K(select_items)); - } else if (OB_FAIL(create_simple_column_expr(target_table->get_table_name(), select_item.alias_name_, target_table->table_id_, target_expr))) { + LOG_WARN("unexpected select item", K(ret), K(i), K(orig_select_items)); + } else if (!ObOptimizerUtil::find_item(orig_group_by_exprs, orig_expr) + && T_FUN_COUNT != orig_expr->get_expr_type()) { + select_item.expr_ = NULL; + } else if (OB_FAIL(create_simple_column_expr(table_name, select_item.alias_name_, table_id, select_item.expr_))) { LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (!ObOptimizerUtil::find_item(orig_group_by_exprs, expr) - && T_FUN_COUNT != expr->get_expr_type()) { - target_columns.at(i) = static_cast(target_expr); - values_exprs.at(i) = NULL; - } else if (OB_FAIL(create_simple_column_expr(source_table->get_table_name(), select_item.alias_name_, source_table->table_id_, source_expr))) { - LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(copier.add_replaced_expr(expr, source_expr))) { + } else if (OB_FAIL(copier.add_replaced_expr(orig_expr, select_item.expr_))) { LOG_WARN("failed to add replace pair", K(ret)); - } else { - target_columns.at(i) = static_cast(target_expr); - values_exprs.at(i) = source_expr; } } // 2. add sum aggr - for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { ObRawExpr *source_count = NULL; - ObRawExpr *target_count = NULL; - if (OB_ISNULL(expr = select_items.at(i).expr_)) { + SelectItem &select_item = select_items.at(i); + int64_t idx = OB_INVALID_INDEX; + if (OB_ISNULL(orig_expr = orig_select_items.at(i).expr_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected select item", K(ret), K(i), K(select_items)); - } else if (NULL != values_exprs.at(i) || T_FUN_SUM != expr->get_expr_type()) { + LOG_WARN("unexpected select item", K(ret), K(i), K(orig_select_items)); + } else if (NULL != select_item.expr_ || T_FUN_SUM != orig_expr->get_expr_type()) { /* do nothing */ - } else if (OB_FAIL(create_simple_column_expr(source_table->get_table_name(), select_items.at(i).alias_name_, source_table->table_id_, source_expr))) { - LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(get_dependent_aggr_of_fun_sum(expr, select_items, target_columns, values_exprs, - target_count, source_count))) { + } else if (OB_FAIL(get_dependent_aggr_of_fun_sum(orig_expr, orig_select_items, idx))) { LOG_WARN("failed to get dependent aggr of fun sum", K(ret)); - } else if (OB_FAIL(gen_calc_expr_for_insert_clause_sum(source_count, source_expr, values_exprs.at(i)))) { + } else if (OB_UNLIKELY(0 > idx || select_items.count() <= idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx", K(ret), K(idx), K(select_items.count())); + } else if (OB_FAIL(create_simple_column_expr(table_name, orig_select_items.at(i).alias_name_, table_id, expr))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(gen_calc_expr_for_insert_clause_sum(select_items.at(idx).expr_, expr, select_item.expr_))) { LOG_WARN("failed to gen calc expr for aggr sum", K(ret)); - } else if (OB_FAIL(copier.add_replaced_expr(expr, values_exprs.at(i)))) { + } else if (OB_FAIL(copier.add_replaced_expr(orig_expr, select_item.expr_))) { LOG_WARN("failed to add replace pair", K(ret)); } } @@ -534,8 +847,8 @@ int ObMVPrinter::gen_insert_values_and_desc(const TableItem *target_table, } } for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { - if (NULL == values_exprs.at(i) && - OB_FAIL(copier.copy_on_replace(select_items.at(i).expr_, values_exprs.at(i)))) { + if (NULL == select_items.at(i).expr_ && + OB_FAIL(copier.copy_on_replace(orig_select_items.at(i).expr_, select_items.at(i).expr_))) { LOG_WARN("failed to generate group by exprs", K(ret)); } } @@ -606,32 +919,17 @@ int ObMVPrinter::gen_calc_expr_for_update_clause_sum(ObRawExpr *target_count, int ObMVPrinter::get_dependent_aggr_of_fun_sum(const ObRawExpr *expr, const ObIArray &select_items, - const ObIArray &target_columns, - const ObIArray &values_exprs, - ObRawExpr *&target_count, - ObRawExpr *&source_count) + int64_t &idx) { int ret = OB_SUCCESS; - target_count = NULL; - source_count = NULL; + idx = OB_INVALID_INDEX; const ObAggFunRawExpr *dep_aggr = NULL; if (OB_FAIL(ObMVChecker::get_dependent_aggr_of_fun_sum(mv_checker_.get_stmt(), static_cast(expr), dep_aggr))) { LOG_WARN("failed to get dependent aggr of fun sum", K(ret)); - } else { - int64_t idx = OB_INVALID_INDEX; - for (int64_t i = 0; OB_INVALID_INDEX == idx && OB_SUCC(ret) && i < select_items.count(); ++i) { - if (select_items.at(i).expr_ == dep_aggr) { - idx = i; - } - } - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(0 > idx || values_exprs.count() <= idx || target_columns.count() <= idx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected idx", K(ret), K(idx), K(values_exprs.count()), K(target_columns.count())); - } else if (OB_ISNULL(target_count = target_columns.at(idx)) - || OB_ISNULL(source_count = values_exprs.at(idx))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL", K(ret), K(target_count), K(source_count)); + } + for (int64_t i = 0; OB_INVALID_INDEX == idx && OB_SUCC(ret) && i < select_items.count(); ++i) { + if (select_items.at(i).expr_ == dep_aggr) { + idx = i; } } return ret; @@ -654,8 +952,6 @@ int ObMVPrinter::gen_update_assignments(const ObIArray &tar ObRawExprCopier copier(expr_factory_); ObRawExpr *expr = NULL; ObOpRawExpr *op_expr = NULL; - ObRawExpr *source_count = NULL; - ObRawExpr *target_count = NULL; ObRawExpr *source_sum = NULL; const ObIArray &group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); @@ -692,19 +988,23 @@ int ObMVPrinter::gen_update_assignments(const ObIArray &tar // 2. add sum aggr for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + int64_t idx = OB_INVALID_INDEX; assign.column_expr_ = target_columns.at(i); if (OB_ISNULL(expr = select_items.at(i).expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected select item", K(ret), K(i), K(select_items)); } else if (T_FUN_SUM != expr->get_expr_type()) { /* do nothing */ + } else if (OB_FAIL(get_dependent_aggr_of_fun_sum(expr, select_items, idx))) { + LOG_WARN("failed to get dependent aggr of fun sum", K(ret)); + } else if (OB_UNLIKELY(0 > idx || values_exprs.count() <= idx || target_columns.count() <= idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx", K(ret), K(idx), K(values_exprs.count()), K(target_columns.count())); } else if (OB_FAIL(create_simple_column_expr(source_table->get_table_name(), select_items.at(i).alias_name_, source_table->table_id_, source_sum))) { LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(get_dependent_aggr_of_fun_sum(expr, select_items, target_columns, values_exprs, - target_count, source_count))) { - LOG_WARN("failed to get dependent aggr of fun sum", K(ret)); - } else if (OB_FAIL(gen_calc_expr_for_update_clause_sum(target_count, source_count, target_columns.at(i), + } else if (OB_FAIL(gen_calc_expr_for_update_clause_sum(target_columns.at(idx), values_exprs.at(idx), + target_columns.at(i), source_sum, assign.expr_))) { LOG_WARN("failed to gen calc expr for aggr sum", K(ret)); } else if (OB_FAIL(inner_assigns.push_back(assign))) { @@ -1091,7 +1391,6 @@ int ObMVPrinter::gen_delta_table_view(const TableItem &source_table, ObSelectStm } else if (OB_ISNULL(view_stmt) || OB_ISNULL(mlog_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(view_stmt), K(mlog_schema), K(source_table)); - } else if (OB_FALSE_IT(view_stmt->set_query_ctx(stmt_factory_.get_query_ctx()))) { } else if (OB_FAIL(create_simple_table_item(view_stmt, mlog_schema->get_table_name_str(), table_item))) { LOG_WARN("failed to create simple table item", K(ret)); } else if (OB_FAIL(gen_delta_table_view_conds(*table_item, view_stmt->get_condition_exprs()))) { @@ -1111,15 +1410,21 @@ int ObMVPrinter::gen_delta_table_view_conds(const TableItem &table, ObIArray &conds) { int ret = OB_SUCCESS; + conds.reuse(); ObRawExpr *scn_column = NULL; - if (OB_FAIL(conds.prepare_allocate(2))) { - LOG_WARN("failed to prepare allocate where conds", K(ret)); - } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), ObString("ORA_ROWSCN"), 0, scn_column))) { + ObRawExpr *filter = NULL; + if (OB_FAIL(create_simple_column_expr(table.get_table_name(), ObString("ORA_ROWSCN"), 0, scn_column))) { LOG_WARN("failed to create simple column expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_GT, scn_column, exprs_.last_refresh_scn_, conds.at(0)))) { + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_GT, scn_column, exprs_.last_refresh_scn_, filter))) { LOG_WARN("failed to build greater op expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_LE, scn_column, exprs_.refresh_scn_, conds.at(1)))) { + } else if (OB_FAIL(conds.push_back(filter))) { + LOG_WARN("failed to pushback", K(ret)); + } else if (for_rt_expand_) { + /* do nothing */ + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_LE, scn_column, exprs_.refresh_scn_, filter))) { LOG_WARN("failed to build less or equal op expr", K(ret)); + } else if (OB_FAIL(conds.push_back(filter))) { + LOG_WARN("failed to pushback", K(ret)); } return ret; } @@ -1331,17 +1636,415 @@ int ObMVPrinter::init(const share::SCN &last_refresh_scn, } else if (OB_FAIL(ObRawExprUtils::build_const_string_expr(expr_factory_, ObVarcharType, ObString("N"), cs_type, exprs_.str_n_)) || OB_FAIL(ObRawExprUtils::build_const_string_expr(expr_factory_, ObVarcharType, ObString("O"), cs_type, exprs_.str_o_))) { LOG_WARN("fail to build const string expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_const_uint_expr(expr_factory_, ObUInt64Type, last_refresh_scn.get_val_for_sql(), exprs_.last_refresh_scn_)) - || OB_FAIL(ObRawExprUtils::build_const_uint_expr(expr_factory_, ObUInt64Type, refresh_scn.get_val_for_sql(), exprs_.refresh_scn_))) { - LOG_WARN("failed to build const uint expr", K(ret)); } else if (OB_FAIL(ObRawExprUtils::build_null_expr(expr_factory_, exprs_.null_expr_))) { LOG_WARN("failed to create const null expr", K(ret)); + } else if (for_rt_expand_) { + ObSysFunRawExpr *sys_last_refresh_scn = NULL; + if (OB_FAIL(expr_factory_.create_raw_expr(T_FUN_SYS_LAST_REFRESH_SCN, sys_last_refresh_scn))) { + LOG_WARN("failed to create last_refresh_scn sys expr", K(ret)); + } else { + sys_last_refresh_scn->set_mview_id(mv_schema_.get_table_id()); + sys_last_refresh_scn->set_func_name(ObString::make_string(N_SYS_LAST_REFRESH_SCN)); + exprs_.last_refresh_scn_ = sys_last_refresh_scn; + } } else { + ObConstRawExpr *const_last_refresh_scn = NULL; + if (OB_FAIL(ObRawExprUtils::build_const_uint_expr(expr_factory_, ObUInt64Type, last_refresh_scn.get_val_for_sql(), const_last_refresh_scn)) + || OB_FAIL(ObRawExprUtils::build_const_uint_expr(expr_factory_, ObUInt64Type, refresh_scn.get_val_for_sql(), exprs_.refresh_scn_))) { + LOG_WARN("failed to build const uint expr", K(ret)); + } else { + exprs_.last_refresh_scn_ = const_last_refresh_scn; + } + } + + if (OB_SUCC(ret)) { mv_db_name_ = db_schema->get_database_name_str(); inited_ = true; } return ret; } +int ObMVPrinter::gen_delete_for_simple_mjv(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + ObDeleteStmt *base_del_stmt = NULL; + ObDeleteStmt *del_stmt = NULL; + TableItem *mv_table = NULL; + ObSEArray fake_sel_exprs; + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + if (OB_FAIL(create_simple_stmt(base_del_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(base_del_stmt, mv_schema_.get_table_name(), mv_table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(fake_sel_exprs.prepare_allocate(orig_select_items.count()))) { + LOG_WARN("failed to prepare allocate arrays", K(ret)); + } else { + mv_table->database_name_ = mv_db_name_; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + ObRawExpr *semi_filter = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), orig_select_items.at(i).alias_name_, + mv_table->table_id_, fake_sel_exprs.at(i)))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + if (OB_FAIL(gen_exists_cond_for_mjv(fake_sel_exprs, orig_table_items.at(i), true, semi_filter))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (orig_table_items.count() - 1 == i) { + if (OB_FAIL(base_del_stmt->get_condition_exprs().push_back(semi_filter))) { + LOG_WARN("failed to push back semi filter", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(base_del_stmt))) { + LOG_WARN("failed to push back delete stmt", K(ret)); + } + } else if (OB_FAIL(create_simple_stmt(del_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(del_stmt->get_table_items().assign(base_del_stmt->get_table_items())) + || OB_FAIL(del_stmt->get_from_items().assign(base_del_stmt->get_from_items()))) { + LOG_WARN("failed to assign structure", K(ret)); + } else if (OB_FAIL(del_stmt->get_condition_exprs().push_back(semi_filter))) { + LOG_WARN("failed to push back semi filter", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(del_stmt))) { + LOG_WARN("failed to push back delete stmt", K(ret)); + } + } + } + return ret; +} + +int ObMVPrinter::gen_insert_into_select_for_simple_mjv(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + TableItem *target_table = NULL; + TableItem *source_table = NULL; + ObInsertStmt *base_insert_stmt = NULL; + ObSEArray access_delta_stmts; + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + if (OB_FAIL(gen_access_delta_data_for_simple_mjv(access_delta_stmts))) { + LOG_WARN("failed to generate ", K(ret)); + } else if (OB_UNLIKELY(orig_table_items.count() != access_delta_stmts.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(orig_table_items.count()), K(access_delta_stmts.count())); + } else if (OB_FAIL(create_simple_stmt(base_insert_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(base_insert_stmt, mv_schema_.get_table_name(), target_table, NULL, false))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else { + target_table->database_name_ = mv_db_name_; + ObRawExpr *target_col = NULL; + ObInsertStmt *insert_stmt = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + if (OB_FAIL(create_simple_column_expr(target_table->get_table_name(), orig_select_items.at(i).alias_name_, + target_table->table_id_, target_col))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(base_insert_stmt->get_values_desc().push_back(static_cast(target_col)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + if (orig_table_items.count() - 1 == i) { + insert_stmt = base_insert_stmt; + } else if (OB_FAIL(create_simple_stmt(insert_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(insert_stmt->get_values_desc().assign(base_insert_stmt->get_values_desc())) + || OB_FAIL(insert_stmt->get_table_items().assign(base_insert_stmt->get_table_items()))) { + LOG_WARN("failed to assign structure", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(create_simple_table_item(insert_stmt, DELTA_MAV_VIEW_NAME, source_table, access_delta_stmts.at(i)))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(insert_stmt))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + return ret; +} + +int ObMVPrinter::gen_access_mv_data_for_simple_mjv(ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt = NULL; + TableItem *mv_table = NULL; + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + if (OB_FAIL(create_simple_stmt(sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(sel_stmt, mv_schema_.get_table_name(), mv_table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(sel_stmt->get_select_items().prepare_allocate(orig_select_items.count())) + || OB_FAIL(sel_stmt->get_condition_exprs().prepare_allocate(orig_table_items.count()))) { + LOG_WARN("failed to prepare allocate arrays", K(ret)); + } else { + mv_table->database_name_ = mv_db_name_; + ObIArray &select_items = sel_stmt->get_select_items(); + ObIArray &conds = sel_stmt->get_condition_exprs(); + ObSEArray select_exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), orig_select_items.at(i).alias_name_, + mv_table->table_id_, select_items.at(i).expr_))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(select_exprs.push_back(select_items.at(i).expr_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + if (OB_FAIL(gen_exists_cond_for_mjv(select_exprs, orig_table_items.at(i), false, conds.at(i)))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } + } + } + return ret; +} + +// generate: not exists (select 1 from mlog$_t1 t1 where mv.t1_pk = t1.pk and ora_rowscn > last_refresh_scn(mv_id)) +int ObMVPrinter::gen_exists_cond_for_mjv(const ObIArray &upper_sel_exprs, + const TableItem *source_table, + bool is_exists, + ObRawExpr *&exists_expr) +{ + int ret = OB_SUCCESS; + exists_expr = NULL; + ObOpRawExpr *exists_op_expr = NULL; + ObQueryRefRawExpr *query_ref_expr = NULL; + const ObTableSchema *mlog_schema = NULL; + ObSelectStmt *subquery = NULL; + TableItem *mlog_table = NULL; + SelectItem sel_item; + sel_item.expr_ = exprs_.int_one_; + const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); + if (OB_UNLIKELY(upper_sel_exprs.count() != select_items.count()) || OB_ISNULL(source_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(upper_sel_exprs.count()), K(select_items.count()), K(source_table)); + } else if (OB_FAIL(create_simple_stmt(subquery))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(mv_checker_.get_mlog_table_schema(source_table, mlog_schema))) { + LOG_WARN("failed to get mlog schema", K(ret), KPC(source_table)); + } else if (OB_FAIL(expr_factory_.create_raw_expr(T_REF_QUERY, query_ref_expr)) + || OB_FAIL(expr_factory_.create_raw_expr(is_exists ? T_OP_EXISTS : T_OP_NOT_EXISTS, exists_op_expr))) { + LOG_WARN("failed to create raw expr", K(ret)); + } else if (OB_ISNULL(query_ref_expr) || OB_ISNULL(exists_op_expr) || OB_ISNULL(mlog_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL", K(ret), K(query_ref_expr), K(exists_op_expr), K(mlog_schema)); + } else if (OB_FAIL(exists_op_expr->add_param_expr(query_ref_expr))) { + LOG_WARN("failed to add param expr", K(ret)); + } else if (OB_FAIL(create_simple_table_item(subquery, mlog_schema->get_table_name_str(), mlog_table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(subquery->get_select_items().push_back(sel_item))) { + LOG_WARN("failed to push back not exists expr", K(ret)); + } else if (OB_FAIL(gen_delta_table_view_conds(*mlog_table, subquery->get_condition_exprs()))) { + LOG_WARN("failed to generate delta table view conds", K(ret)); + } else { + exists_expr = exists_op_expr; + mlog_table->database_name_ = source_table->database_name_; + query_ref_expr->set_ref_stmt(subquery); + ObRawExpr *expr = NULL; + ObColumnRefRawExpr *col_expr = NULL; + ObSEArray rowkeys; + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + int64_t idx = OB_INVALID_INDEX; + if (OB_ISNULL(expr = select_items.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(select_items)); + } else if (!expr->is_column_ref_expr()) { + /* do nothing */ + } else if (OB_FALSE_IT(col_expr = static_cast(expr))) { + } else if (!col_expr->is_rowkey_column() || col_expr->get_table_id() != source_table->table_id_) { + /* do nothing */ + } else if (OB_FAIL(add_var_to_array_no_dup(rowkeys, expr, &idx))) { + LOG_WARN("failed to add_var to array no dup", K(ret)); + } else if (OB_INVALID_INDEX == idx) { + /* do nothing */ + } else if (OB_FAIL(create_simple_column_expr(mlog_table->get_table_name(), col_expr->get_column_name(), mlog_table->table_id_, expr))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, upper_sel_exprs.at(i), expr, expr))) { + LOG_WARN("failed to build equal expr", K(ret)); + } else if (OB_FAIL(subquery->get_condition_exprs().push_back(expr))) { + LOG_WARN("failed to push back null safe equal expr", K(ret)); + } + } + } + return ret; +} + +int ObMVPrinter::gen_access_delta_data_for_simple_mjv(ObIArray &access_delta_stmts) +{ + int ret = OB_SUCCESS; + ObSelectStmt *base_delta_stmt = NULL; + ObSEArray semi_filters; + ObSEArray anti_filters; + const int64_t table_size = mv_checker_.get_stmt().get_table_items().count(); + if (OB_UNLIKELY(table_size < 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(table_size)); + } else if (OB_FAIL(access_delta_stmts.prepare_allocate(table_size))) { + LOG_WARN("failed to prepare allocate arrays", K(ret), K(table_size)); + } else if (OB_FAIL(prepare_gen_access_delta_data_for_simple_mjv(base_delta_stmt, + semi_filters, + anti_filters))) { + LOG_WARN("failed to prepare generate access delta data for simple_mjv", K(ret)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < table_size - 1; ++i) { + if (OB_FAIL(gen_one_access_delta_data_for_simple_mjv(*base_delta_stmt, i, semi_filters, anti_filters, + access_delta_stmts.at(i)))) { + LOG_WARN("failed to generate one access delta data for simple_mjv", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(base_delta_stmt->get_condition_exprs().push_back(semi_filters.at(table_size - 1)))) { + LOG_WARN("failed to push back", K(ret)); + } else { + access_delta_stmts.at(table_size - 1) = base_delta_stmt; + } + return ret; +} + +int ObMVPrinter::prepare_gen_access_delta_data_for_simple_mjv(ObSelectStmt *&base_delta_stmt, + ObIArray &semi_filters, + ObIArray &anti_filters) +{ + int ret = OB_SUCCESS; + base_delta_stmt = NULL; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + if (OB_FAIL(semi_filters.prepare_allocate(orig_table_items.count())) + || OB_FAIL(anti_filters.prepare_allocate(orig_table_items.count()))) { + LOG_WARN("failed to prepare allocate arrays", K(ret), K(orig_table_items.count())); + } else if (OB_FAIL(create_simple_stmt(base_delta_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else { + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + ObIArray &select_items = base_delta_stmt->get_select_items(); + ObIArray &cur_table_items = base_delta_stmt->get_table_items(); + ObSEArray column_items; + SelectItem sel_item; + ObRawExpr *old_col = NULL; + ObRawExpr *new_col = NULL; + const TableItem *orig_table = NULL; + TableItem *table = NULL; + ObRawExprCopier copier(expr_factory_); + ObSemiToInnerHint *semi_to_inner_hint = NULL; + ObSEArray conflict_hints; + ObSEArray select_exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + column_items.reuse(); + if (OB_ISNULL(orig_table = orig_table_items.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(orig_table_items)); + } else if (OB_FAIL(mv_checker_.get_stmt().get_column_items(orig_table->table_id_, column_items))) { + LOG_WARN("failed to get column items", K(ret)); + } else if (OB_FAIL(create_simple_table_item(base_delta_stmt, orig_table->table_name_, table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else { + table->alias_name_ = orig_table->alias_name_; + table->synonym_name_ = orig_table->synonym_name_; + table->database_name_ = orig_table->database_name_; + table->synonym_db_name_ = orig_table->synonym_db_name_; + if (!for_rt_expand_) { + table->flashback_query_expr_ = exprs_.refresh_scn_; + table->flashback_query_type_ = TableItem::USING_SCN; + } + } + for (int64_t j = 0; OB_SUCC(ret) && j < column_items.count(); ++j) { + if (OB_FAIL(create_simple_column_expr(table->get_table_name(), column_items.at(j).column_name_, + table->table_id_, new_col))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else if (OB_FAIL(copier.add_replaced_expr(column_items.at(j).expr_, new_col))) { + LOG_WARN("failed to add replace pair", K(ret)); + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + sel_item.is_real_alias_ = true; + sel_item.alias_name_ = orig_select_items.at(i).alias_name_; + if (OB_FAIL(copier.copy_on_replace(orig_select_items.at(i).expr_, sel_item.expr_))) { + LOG_WARN("failed to generate group by exprs", K(ret)); + } else if (OB_FAIL(select_items.push_back(sel_item)) + || OB_FAIL(select_exprs.push_back(sel_item.expr_))) { + LOG_WARN("failed to pushback", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + if (OB_FAIL(gen_exists_cond_for_mjv(select_exprs, orig_table_items.at(i), true, semi_filters.at(i)))) { + LOG_WARN("failed to generate exists filter", K(ret)); + } else if (OB_FAIL(gen_exists_cond_for_mjv(select_exprs, orig_table_items.at(i), false, anti_filters.at(i)))) { + LOG_WARN("failed to generate not exists filter", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObQueryHint::create_hint(&alloc_, T_SEMI_TO_INNER, semi_to_inner_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(base_delta_stmt->get_stmt_hint().merge_hint(*semi_to_inner_hint, + ObHintMergePolicy::HINT_DOMINATED_EQUAL, + conflict_hints))) { + LOG_WARN("failed to merge hint", K(ret)); + } else if (OB_FAIL(base_delta_stmt->deep_copy_join_tables(alloc_, copier, mv_checker_.get_stmt()))) { + LOG_WARN("failed to deep copy join tables", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), base_delta_stmt->get_condition_exprs()))) { + LOG_WARN("failed to deep copy where conditions", K(ret)); + } else if (OB_FAIL(base_delta_stmt->get_from_items().assign(mv_checker_.get_stmt().get_from_items()))) { + LOG_WARN("failed to assign from items", K(ret)); + } else { + // for non joined table, adjust table id in from item + ObIArray &from_items = base_delta_stmt->get_from_items(); + int64_t idx = OB_INVALID_INDEX; + for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { + if (from_items.at(i).is_joined_) { + /* do nothing */ + } else if (OB_FAIL(mv_checker_.get_stmt().get_table_item_idx(from_items.at(i).table_id_, idx))) { + LOG_WARN("failed to get table item", K(ret)); + } else if (OB_UNLIKELY(idx < 0 || idx >= base_delta_stmt->get_table_size()) + || OB_ISNULL(base_delta_stmt->get_table_item(idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx", K(ret), K(idx), K(base_delta_stmt->get_table_size())); + } else { + from_items.at(i).table_id_ = base_delta_stmt->get_table_item(idx)->table_id_; + } + } + } + } + return ret; +} + +// delta_mv = delta_t1 join pre_t2 join pre_t3 ... join pre_tn +// union all t1 join delta_t2 join pre_t3 ... join pre_tn +// ... +// union all t1 join t2 join t3 ... join delta_tn +// input table_idx specify the delta table +int ObMVPrinter::gen_one_access_delta_data_for_simple_mjv(const ObSelectStmt &base_delta_stmt, + const int64_t table_idx, + const ObIArray &semi_filters, + const ObIArray &anti_filters, + ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt = NULL; + if (OB_UNLIKELY(table_idx < 0 || table_idx >= semi_filters.count() + || semi_filters.count() != anti_filters.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(table_idx), K(semi_filters.count()), K(anti_filters.count())); + } else if (OB_FAIL(create_simple_stmt(sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(sel_stmt->get_joined_tables().assign(base_delta_stmt.get_joined_tables())) + || OB_FAIL(sel_stmt->get_table_items().assign(base_delta_stmt.get_table_items())) + || OB_FAIL(sel_stmt->get_from_items().assign(base_delta_stmt.get_from_items())) + || OB_FAIL(sel_stmt->get_select_items().assign(base_delta_stmt.get_select_items())) + || OB_FAIL(sel_stmt->get_condition_exprs().assign(base_delta_stmt.get_condition_exprs())) + || OB_FAIL(sel_stmt->get_stmt_hint().merge_stmt_hint(base_delta_stmt.get_stmt_hint()))) { + LOG_WARN("failed to assign structure", K(ret)); + } else if (OB_FAIL(sel_stmt->get_condition_exprs().push_back(semi_filters.at(table_idx)))) { + LOG_WARN("failed to push back semi filter", K(ret)); + } else { + for (int64_t i = table_idx + 1; OB_SUCC(ret) && i < anti_filters.count(); ++i) { + if (OB_FAIL(sel_stmt->get_condition_exprs().push_back(anti_filters.at(i)))) { + LOG_WARN("failed to push back anti filter", K(ret)); + } + } + } + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/mv/ob_mv_printer.h b/src/sql/resolver/mv/ob_mv_printer.h index 6f3c15671..5a5655553 100644 --- a/src/sql/resolver/mv/ob_mv_printer.h +++ b/src/sql/resolver/mv/ob_mv_printer.h @@ -44,7 +44,7 @@ struct SharedPrinterRawExprs ObConstRawExpr *int_zero_; ObConstRawExpr *int_one_; ObConstRawExpr *int_neg_one_; - ObConstRawExpr *last_refresh_scn_; + ObRawExpr *last_refresh_scn_; ObConstRawExpr *refresh_scn_; ObConstRawExpr *str_n_; ObConstRawExpr *str_o_; @@ -74,6 +74,7 @@ public: static const ObString DELTA_TABLE_VIEW_NAME; static const ObString DELTA_BASIC_MAV_VIEW_NAME; static const ObString DELTA_MAV_VIEW_NAME; + static const ObString INNER_RT_MV_VIEW_NAME; // column name static const ObString HEAP_TABLE_ROWKEY_COL_NAME; static const ObString OLD_NEW_COL_NAME; @@ -99,7 +100,35 @@ public: private: int init(const share::SCN &last_refresh_scn, const share::SCN &refresh_scn); int gen_mv_operator_stmts(ObIArray &dml_stmts); - int gen_refresh_dmls_for_simple_mav(ObIArray &dml_stmts); + int gen_refresh_dmls_for_mv(ObIArray &dml_stmts); + int gen_real_time_view_for_mv(ObSelectStmt *&sel_stmt); + int gen_delete_insert_for_simple_mjv(ObIArray &dml_stmts); + int gen_insert_into_select_for_simple_mjv(ObIArray &dml_stmts); + int gen_real_time_view_for_simple_mjv(ObSelectStmt *&sel_stmt); + int gen_delete_for_simple_mjv(ObIArray &dml_stmts); + int gen_access_mv_data_for_simple_mjv(ObSelectStmt *&sel_stmt); + int gen_exists_cond_for_mjv(const ObIArray &upper_sel_exprs, + const TableItem *source_table, + bool is_exists, + ObRawExpr *&exists_expr); + int gen_access_delta_data_for_simple_mjv(ObIArray &access_delta_stmts); + int prepare_gen_access_delta_data_for_simple_mjv(ObSelectStmt *&base_delta_stmt, + ObIArray &semi_filters, + ObIArray &anti_filters); + int gen_one_access_delta_data_for_simple_mjv(const ObSelectStmt &base_delta_stmt, + const int64_t table_idx, + const ObIArray &semi_filters, + const ObIArray &anti_filters, + ObSelectStmt *&sel_stmt); + int gen_real_time_view_for_simple_mav(ObSelectStmt *&sel_stmt); + int gen_real_time_view_scn_cte(ObSelectStmt &root_stmt); + int gen_real_time_view_filter_for_mav(ObSelectStmt &sel_stmt); + int gen_inner_real_time_view_for_mav(ObSelectStmt &sel_stmt, TableItem *&view_table); + int gen_inner_real_time_view_tables_for_mav(ObSelectStmt &sel_stmt); + int gen_inner_real_time_view_select_list_for_mav(ObSelectStmt &sel_stmt); + int gen_select_items_for_mav(const ObString &table_name, + const uint64_t table_id, + ObIArray &select_items); int gen_merge_for_simple_mav(ObMergeStmt *&merge_stmt); int gen_update_insert_delete_for_simple_mav(ObIArray &dml_stmts); int gen_insert_for_mav(ObSelectStmt *delta_mv_stmt, @@ -137,10 +166,7 @@ private: ObRawExpr *&calc_sum); int get_dependent_aggr_of_fun_sum(const ObRawExpr *expr, const ObIArray &select_items, - const ObIArray &target_columns, - const ObIArray &values_exprs, - ObRawExpr *&target_count, - ObRawExpr *&source_count); + int64_t &idx); int gen_update_assignments(const ObIArray &target_columns, const ObIArray &values_exprs, const TableItem *source_table, @@ -214,6 +240,7 @@ private: ObStmtFactory &stmt_factory_; ObRawExprFactory &expr_factory_; SharedPrinterRawExprs exprs_; + DISALLOW_COPY_AND_ASSIGN(ObMVPrinter); }; } diff --git a/src/sql/resolver/mv/ob_mv_provider.cpp b/src/sql/resolver/mv/ob_mv_provider.cpp index e5f6dc7a6..117521cbd 100644 --- a/src/sql/resolver/mv/ob_mv_provider.cpp +++ b/src/sql/resolver/mv/ob_mv_provider.cpp @@ -37,6 +37,7 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, { int ret = OB_SUCCESS; dependency_infos_.reuse(); + operators_.reuse(); if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("mv provider is inited twice", K(ret)); @@ -57,6 +58,8 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, const ObTableSchema *mv_schema = NULL; ObQueryCtx *query_ctx = NULL; int64_t max_version = OB_INVALID_VERSION; + ObSEArray operators; + ObSEArray dependency_infos; if (OB_ISNULL(query_ctx = stmt_factory.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(query_ctx)); @@ -74,11 +77,13 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, view_stmt))) { LOG_WARN("failed to gen mv stmt", K(ret)); } else if (OB_FAIL(ObDependencyInfo::collect_dep_infos(query_ctx->reference_obj_tables_, - dependency_infos_, + dependency_infos, ObObjectType::VIEW, OB_INVALID_ID, max_version))) { LOG_WARN("failed to collect dep infos", K(ret)); + } else if (OB_FAIL(dependency_infos_.assign(dependency_infos))) { + LOG_WARN("failed to assign fixed array", K(ret)); } else if (OB_FAIL(check_mv_column_type(mv_schema, view_stmt))) { if (OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH == ret) { inited_ = true; @@ -94,9 +99,11 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, stmt_factory, expr_factory, session_info, - operators_, + operators, refreshable_type_))) { LOG_WARN("failed to print mv operators", K(ret)); + } else if (OB_FAIL(operators_.assign(operators))) { + LOG_WARN("failed to assign fixed array", K(ret)); } else { inited_ = true; } @@ -125,13 +132,16 @@ int ObMVProvider::check_mv_refreshable(bool &can_fast_refresh) const return ret; } -int ObMVProvider::get_operators(const ObIArray *&operators) const +int ObMVProvider::get_fast_refresh_operators(const ObIArray *&operators) const { int ret = OB_SUCCESS; operators = NULL; - if (OB_UNLIKELY(!inited_ || ObMVRefreshableType::OB_MV_COMPLETE_REFRESH >= refreshable_type_)) { + if (OB_UNLIKELY(!inited_ || for_rt_expand_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected call for get_operators", K(ret), K(inited_), K(refreshable_type_)); + LOG_WARN("unexpected call for get_operators", K(ret), K(inited_), K(for_rt_expand_)); + } else if (OB_UNLIKELY(ObMVRefreshableType::OB_MV_COMPLETE_REFRESH >= refreshable_type_)) { + ret = OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH; + LOG_WARN("mview can not fast refresh", K(ret), K(mview_id_)); } else if (OB_UNLIKELY(operators_.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected empty operators", K(ret)); @@ -141,6 +151,28 @@ int ObMVProvider::get_operators(const ObIArray *&operators) const return ret; } +// expand_view will used to generate plan, need use alloc to deep copy the query str +int ObMVProvider::get_real_time_mv_expand_view(ObIAllocator &alloc, ObString &expand_view) const +{ + int ret = OB_SUCCESS; + expand_view.reset(); + if (OB_UNLIKELY(!inited_ || !for_rt_expand_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected call for get_operators", K(ret), K(inited_), K(for_rt_expand_)); + } else if (OB_UNLIKELY(ObMVRefreshableType::OB_MV_COMPLETE_REFRESH >= refreshable_type_)) { + ret = OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH; + LOG_WARN("mview can not on query computation", K(ret), K(mview_id_)); + } else if (OB_UNLIKELY(1 != operators_.count() || operators_.at(0).empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty operators", K(ret)); + } else if (OB_FAIL(ob_write_string(alloc, operators_.at(0), expand_view))) { + LOG_WARN("failed to write string", K(ret)); + } else { + LOG_TRACE("finish generate rt mv expand view", K(mview_id_), K(expand_view)); + } + return ret; +} + int ObMVProvider::get_mv_dependency_infos(ObIArray &dep_infos) const { int ret = OB_SUCCESS; @@ -174,6 +206,8 @@ int ObMVProvider::check_mv_column_type(const ObTableSchema *mv_schema, || OB_ISNULL(select_items.at(i).expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(i), K(org_column), K(select_items.at(i))); + } else if (select_items.at(i).expr_->is_const_expr()) { + /* do nothing */ } else if (OB_FAIL(ObCreateViewResolver::fill_column_meta_infos(*select_items.at(i).expr_, mv_schema->get_charset_type(), mv_schema->get_table_id(), @@ -209,7 +243,12 @@ int ObMVProvider::check_mv_column_type(const ObColumnSchemaV2 &org_column, } } - if (OB_SUCC(ret) && !is_valid_col) { + if (OB_FAIL(ret) || is_valid_col) { + } else if (org_column.get_meta_type().is_number() + && cur_column.get_meta_type().is_decimal_int() + && lib::is_oracle_mode()) { + /* in oracle mode, const number is resolved as decimal int except ddl stmt */ + } else { ret = OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH; LOG_WARN("mv column changed", K(ret), K(org_column), K(cur_column)); } @@ -240,6 +279,7 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, resolver_ctx.stmt_factory_ = &stmt_factory; resolver_ctx.sql_proxy_ = GCTX.sql_proxy_; resolver_ctx.query_ctx_ = stmt_factory.get_query_ctx(); + ObSelectStmt *sel_stmt = NULL; ObSelectResolver select_resolver(resolver_ctx); if (OB_ISNULL(resolver_ctx.query_ctx_)) { ret = OB_ERR_UNEXPECTED; @@ -263,10 +303,13 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ = static_cast(parse_result.question_mark_ctx_.count_))) { } else if (OB_FAIL(select_resolver.resolve(*node))) { LOG_WARN("resolve view definition failed", K(ret)); - } else if (OB_ISNULL(view_stmt = static_cast(select_resolver.get_basic_stmt()))) { + } else if (OB_ISNULL(sel_stmt = static_cast(select_resolver.get_basic_stmt()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mv stmt", K(ret), K(view_stmt)); + LOG_WARN("invalid mv stmt", K(ret), K(sel_stmt)); + } else if (OB_FAIL(sel_stmt->formalize_stmt_expr_reference(&expr_factory, &session_info, true))) { + LOG_WARN("failed to formalize stmt reference", K(ret)); } else { + view_stmt = sel_stmt; LOG_DEBUG("generate mv stmt", KPC(view_stmt)); } return ret; diff --git a/src/sql/resolver/mv/ob_mv_provider.h b/src/sql/resolver/mv/ob_mv_provider.h index cfb133cab..dc9a4009b 100644 --- a/src/sql/resolver/mv/ob_mv_provider.h +++ b/src/sql/resolver/mv/ob_mv_provider.h @@ -12,6 +12,7 @@ #ifndef OCEANBASE_SQL_RESOLVER_MV_OB_MV_PROVIDER_H_ #define OCEANBASE_SQL_RESOLVER_MV_OB_MV_PROVIDER_H_ +#include "share/scn.h" #include "lib/string/ob_string.h" #include "lib/hash_func/ob_hash_func.h" #include "lib/container/ob_se_array.h" @@ -32,18 +33,22 @@ public: mview_id_(mview_id), for_rt_expand_(for_rt_expand), inited_(false), - refreshable_type_(OB_MV_REFRESH_INVALID) + refreshable_type_(OB_MV_REFRESH_INVALID), + operators_(&inner_alloc_), + dependency_infos_(&inner_alloc_) {} ~ObMVProvider() {} + int init_mv_provider(ObSchemaGetterGuard *schema_guard, ObSQLSessionInfo *session_info) + { return init_mv_provider(share::SCN(), share::SCN(), schema_guard, session_info); } int init_mv_provider(const share::SCN &last_refresh_scn, const share::SCN &refresh_scn, ObSchemaGetterGuard *schema_guard, ObSQLSessionInfo *session_info); int check_mv_refreshable(bool &can_fast_refresh) const; - int get_operators(const ObIArray *&operators) const; + int get_fast_refresh_operators(const ObIArray *&operators) const; + int get_real_time_mv_expand_view(ObIAllocator &alloc, ObString &expand_view) const; int get_mv_dependency_infos(ObIArray &dep_infos) const; - private: int check_mv_column_type(const ObTableSchema *mv_schema, const ObSelectStmt *view_stmt); int check_mv_column_type(const ObColumnSchemaV2 &org_column, const ObColumnSchemaV2 &cur_column); @@ -65,8 +70,8 @@ private: const bool for_rt_expand_; bool inited_; ObMVRefreshableType refreshable_type_; - ObSEArray operators_; // refresh or real time access operator for mv - ObSEArray dependency_infos_; + common::ObFixedArray operators_; // refresh or real time access operator for mv + common::ObFixedArray dependency_infos_; }; } diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 4d98775f0..332c8b6cd 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -1211,8 +1211,9 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS OZ( (static_cast(stmt)->disable_writing_external_table()) ); } - if (OB_SUCC(ret) && stmt->is_dml_stmt() - && !params_.session_info_->is_inner()) { + if (OB_SUCC(ret) && !params_.session_info_->is_inner() + && stmt->is_dml_stmt() && !stmt->is_explain_stmt() && 0 == stmt->get_stmt_id()) { + // allowed explain for dml write mv, allowed refresh mv sql write mv OZ( (static_cast(stmt)->disable_writing_materialized_view()) ); } diff --git a/src/sql/resolver/ob_resolver_define.h b/src/sql/resolver/ob_resolver_define.h index 6899d876e..3c6fbb6bd 100644 --- a/src/sql/resolver/ob_resolver_define.h +++ b/src/sql/resolver/ob_resolver_define.h @@ -371,7 +371,8 @@ struct ObResolverParams is_expanding_view_(false), is_resolve_lateral_derived_table_(false), package_guard_(NULL), - star_expansion_infos_() + star_expansion_infos_(), + is_for_rt_mv_(false) {} bool is_force_trace_log() { return force_trace_log_; } @@ -443,6 +444,7 @@ public: bool is_resolve_lateral_derived_table_; // used to mark resolve lateral derived table. pl::ObPLPackageGuard *package_guard_; common::ObArray star_expansion_infos_; + bool is_for_rt_mv_; // call resolve in transformation for expanding inline real-time materialized view }; } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/rewrite/ob_stmt_comparer.cpp b/src/sql/rewrite/ob_stmt_comparer.cpp index 31d6ef0e0..16a974c49 100644 --- a/src/sql/rewrite/ob_stmt_comparer.cpp +++ b/src/sql/rewrite/ob_stmt_comparer.cpp @@ -17,6 +17,7 @@ #include "sql/optimizer/ob_optimizer_util.h" #include "sql/ob_sql_context.h" #include "common/ob_smart_call.h" +#include "objit/expr/ob_iraw_expr.h" using namespace oceanbase::sql; @@ -141,7 +142,7 @@ bool ObStmtCompareContext::compare_column(const ObColumnRefRawExpr &inner, { bool bret = false; int idx = get_table_map_idx(inner.get_table_id(), outer.get_table_id()); - if (inner.get_table_id() == outer.get_table_id()) { + if (is_in_same_stmt_ && inner.get_table_id() == outer.get_table_id()) { bret = inner.get_column_id() == outer.get_column_id(); } else if (idx == OB_INVALID_ID) { //do nothing @@ -208,6 +209,17 @@ bool ObStmtCompareContext::compare_const(const ObConstRawExpr &left, const ObCon } } else if (left.is_param_expr() || right.is_param_expr()) { bret = ObExprEqualCheckContext::compare_const(left, right); + if (bret && (left.get_value().is_unknown() || right.get_value().is_unknown())) { + const ObConstRawExpr &unkonwn_expr = left.get_value().is_unknown() ? left : right; + ObPCConstParamInfo const_param_info; + if (OB_FAIL(const_param_info.const_idx_.push_back(unkonwn_expr.get_value().get_unknown()))) { + LOG_WARN("failed to push back element", K(ret)); + } else if (OB_FAIL(const_param_info.const_params_.push_back(unkonwn_expr.get_result_type().get_param()))) { + LOG_WARN("failed to psuh back param const value", K(ret)); + } else if (OB_FAIL(const_param_info_.push_back(const_param_info))) { + LOG_WARN("failed to push back const param info", K(ret)); + } + } } else { bret = left.get_value().is_equal(right.get_value(), CS_TYPE_BINARY); } @@ -259,7 +271,9 @@ bool ObStmtCompareContext::compare_query(const ObQueryRefRawExpr &first, second_sel, stmt_map_info, relation, - true))) { + true, + true, + is_in_same_stmt_))) { LOG_WARN("failed to compute stmt relationship", K(ret)); err_code_ = ret; } else if (stmt_map_info.is_select_item_equal_ && QueryRelation::QUERY_EQUAL == relation) { @@ -309,15 +323,16 @@ int ObStmtComparer::compute_stmt_overlap(const ObDMLStmt *first, ObStmtMapInfo &map_info) { int ret = OB_SUCCESS; - int64_t match_count = 0; + QueryRelation relation; map_info.reset(); if (OB_ISNULL(first) || OB_ISNULL(second)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("stmts are null", K(ret), K(first), K(second)); } else if (OB_FAIL(compute_from_items_map(first, second, + true, map_info, - match_count))) { + relation))) { LOG_WARN("failed to compute from items map", K(ret)); } else if (OB_FAIL(map_info.cond_map_.prepare_allocate(first->get_condition_size()))) { LOG_WARN("failed to pre-allocate condition map", K(ret)); @@ -327,7 +342,7 @@ int ObStmtComparer::compute_stmt_overlap(const ObDMLStmt *first, second->get_condition_exprs(), map_info, map_info.cond_map_, - match_count))) { + relation))) { LOG_WARN("failed to compute condition map", K(ret)); } else { LOG_TRACE("stmt map info", K(map_info)); @@ -345,6 +360,7 @@ int ObStmtComparer::is_same_from(const ObDMLStmt *first, const FromItem &first_from, const ObDMLStmt *second, const FromItem &second_from, + bool is_in_same_stmt, ObStmtMapInfo &map_info, bool &is_same) { @@ -359,6 +375,7 @@ int ObStmtComparer::is_same_from(const ObDMLStmt *first, first->get_joined_table(first_from.table_id_), second, second->get_joined_table(second_from.table_id_), + is_in_same_stmt, map_info, relation))) { LOG_WARN(" compare joined table item failed", K(ret)); @@ -373,6 +390,7 @@ int ObStmtComparer::is_same_from(const ObDMLStmt *first, first->get_table_item_by_id(first_from.table_id_), second, second->get_table_item_by_id(second_from.table_id_), + is_in_same_stmt, map_info, relation))) { LOG_WARN(" compare table item failed", K(ret)); @@ -389,7 +407,9 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, const ObDMLStmt *second, ObStmtMapInfo &map_info, QueryRelation &relation, - bool is_strict_select_list) + bool is_strict_select_list, + bool need_check_select_items, + bool is_in_same_stmt) { int ret = OB_SUCCESS; int64_t first_count = 0; @@ -402,7 +422,7 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(first), K(second), K(ret)); } else if (!first->is_select_stmt() || !second->is_select_stmt()) { - /*do nothing*/ + LOG_TRACE("failed to compare, not a select item", K(first->is_select_stmt()), K(second->is_select_stmt())); } else if (FALSE_IT(first_sel = const_cast(static_cast(first)))) { /*do nothing*/ } else if (FALSE_IT(second_sel = const_cast(static_cast(second)))) { @@ -410,29 +430,24 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, } else if (first_sel->has_recursive_cte() || second_sel->has_recursive_cte() || first_sel->has_hierarchical_query() || second_sel->has_hierarchical_query() || first_sel->is_contains_assignment() || second_sel->is_contains_assignment()) { - /*do nothing*/ + LOG_TRACE("failed to compare, contain can not compare query"); } else if (first_sel->is_set_stmt() && second_sel->is_set_stmt()) { - if (OB_FAIL(compare_set_stmt(first_sel, second_sel, map_info, relation))) { + if (OB_FAIL(compare_set_stmt(first_sel, second_sel, map_info, relation, is_in_same_stmt))) { LOG_WARN("failed to compare set stmt", K(ret)); } } else if (first_sel->is_set_stmt() || second_sel->is_set_stmt()) { /*do nothing*/ - } else if (first_sel->get_from_item_size() != second_sel->get_from_item_size()) { - /*do nothing*/ + } else if (first_sel->get_from_item_size() != second_sel->get_from_item_size()) { // TODO for the further mv rewrite, from item size may be different + LOG_TRACE("failed to compare, from item size not match", K(first_sel->get_from_item_size()), K(second_sel->get_from_item_size())); } else { // check from items if (OB_FAIL(compute_from_items_map(first_sel, second_sel, + is_in_same_stmt, map_info, - match_count))) { + relation))) { LOG_WARN("failed to compute from items map", K(ret)); - } else if (match_count != first_sel->get_from_item_size()) { - relation = QueryRelation::QUERY_UNCOMPARABLE; - LOG_TRACE("succeed to check from item map", K(relation), K(map_info)); } else { - relation = QueryRelation::QUERY_EQUAL; - map_info.is_table_equal_ = true; - map_info.is_from_equal_ = true; LOG_TRACE("succeed to check from item map", K(relation), K(map_info)); } @@ -442,6 +457,7 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, second_count = second_sel->get_semi_info_size(); if (OB_FAIL(compute_semi_infos_map(first_sel, second_sel, + is_in_same_stmt, map_info, match_count))) { LOG_WARN("failed to compute semi info map", K(ret)); @@ -459,23 +475,39 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, if (OB_SUCC(ret) && QueryRelation::QUERY_EQUAL == relation) { first_count = first_sel->get_condition_size(); second_count = second_sel->get_condition_size(); + QueryRelation this_relation; if (OB_FAIL(compute_conditions_map(first_sel, - second_sel, - first_sel->get_condition_exprs(), - second_sel->get_condition_exprs(), - map_info, - map_info.cond_map_, - match_count))) { + second_sel, + first_sel->get_condition_exprs(), + second_sel->get_condition_exprs(), + map_info, + map_info.cond_map_, + this_relation, + is_in_same_stmt, + false, + true))) { LOG_WARN("failed to compute conditions map", K(ret)); - } else if (match_count == first_count && match_count == second_count) { - relation = QueryRelation::QUERY_EQUAL; + } else if (second_sel->has_group_by() && QueryRelation::QUERY_EQUAL != this_relation) { + relation = QueryRelation::QUERY_UNCOMPARABLE; + LOG_TRACE("succeed to check conditions map", K(relation), K(map_info)); + } else if (QueryRelation::QUERY_EQUAL == this_relation) { map_info.is_cond_equal_ = true; LOG_TRACE("succeed to check conditions map", K(relation), K(map_info)); - } else if (match_count == first_count && match_count < second_count) { - relation = QueryRelation::QUERY_RIGHT_SUBSET; + } else if (QueryRelation::QUERY_LEFT_SUBSET == this_relation) { + if (relation == QueryRelation::QUERY_EQUAL + || relation == QueryRelation::QUERY_LEFT_SUBSET) { + relation = QueryRelation::QUERY_LEFT_SUBSET; + } else { + relation = QueryRelation::QUERY_UNCOMPARABLE; + } LOG_TRACE("succeed to check conditions map", K(relation), K(map_info)); - } else if (match_count < first_count && match_count == second_count) { - relation = QueryRelation::QUERY_LEFT_SUBSET; + } else if (QueryRelation::QUERY_RIGHT_SUBSET == this_relation) { + if (relation == QueryRelation::QUERY_EQUAL + || relation == QueryRelation::QUERY_RIGHT_SUBSET) { + relation = QueryRelation::QUERY_RIGHT_SUBSET; + } else { + relation = QueryRelation::QUERY_UNCOMPARABLE; + } LOG_TRACE("succeed to check conditions map", K(relation), K(map_info)); } else { relation = QueryRelation::QUERY_UNCOMPARABLE; @@ -490,7 +522,14 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, second_count = second_sel->get_group_exprs().count(); int64_t first_rollup_count = first_sel->get_rollup_exprs().count(); int64_t second_rollup_count = second_sel->get_rollup_exprs().count(); - if ((first_sel->get_aggr_item_size() > 0 || + QueryRelation this_relation; + if (second_count == 0 && first_rollup_count == 0 && second_rollup_count == 0 + && (relation == QueryRelation::QUERY_LEFT_SUBSET || relation == QueryRelation::QUERY_EQUAL) + && !need_check_select_items) { + // for mv rewrite + relation = QueryRelation::QUERY_LEFT_SUBSET; + LOG_TRACE("succeed to check group by map", K(relation), K(map_info)); + } else if ((first_sel->get_aggr_item_size() > 0 || second_sel->get_aggr_item_size() > 0) && relation != QueryRelation::QUERY_EQUAL) { relation = QueryRelation::QUERY_UNCOMPARABLE; @@ -520,9 +559,10 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, second_sel->get_group_exprs(), map_info, map_info.group_map_, - match_count))) { + this_relation, + is_in_same_stmt))) { LOG_WARN("failed to compute group by expr map", K(ret)); - } else if (match_count != first_count || match_count != second_count) { + } else if (this_relation != QueryRelation::QUERY_EQUAL) { relation = QueryRelation::QUERY_UNCOMPARABLE; LOG_TRACE("succeed to check group by map", K(relation), K(map_info)); } else if (first_rollup_count != 0) { @@ -549,6 +589,7 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, if (OB_SUCC(ret) && QueryRelation::QUERY_UNCOMPARABLE != relation) { first_count = first_sel->get_having_expr_size(); second_count = second_sel->get_having_expr_size(); + QueryRelation this_relation; if (0 == first_count && 0 == second_count) { map_info.is_having_equal_ = true; } else if (OB_FAIL(compute_conditions_map(first_sel, @@ -557,17 +598,20 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, second_sel->get_having_exprs(), map_info, map_info.having_map_, - match_count))) { + this_relation, + is_in_same_stmt, + false, + true))) { LOG_WARN("failed to compute having expr map", K(ret)); - } else if (match_count == first_count && match_count == second_count) { + } else if (this_relation == QueryRelation::QUERY_EQUAL) { map_info.is_having_equal_ = true; LOG_TRACE("succeed to check having map", K(relation), K(map_info)); - } else if (match_count == first_count && match_count < second_count && + } else if (this_relation == QueryRelation::QUERY_RIGHT_SUBSET && (relation == QueryRelation::QUERY_RIGHT_SUBSET || relation == QueryRelation::QUERY_EQUAL)) { relation = QueryRelation::QUERY_RIGHT_SUBSET; LOG_TRACE("succeed to check having map", K(relation), K(map_info)); - } else if (match_count == second_count && match_count < first_count && + } else if (this_relation == QueryRelation::QUERY_LEFT_SUBSET && (relation == QueryRelation::QUERY_LEFT_SUBSET || relation == QueryRelation::QUERY_EQUAL)) { relation = QueryRelation::QUERY_LEFT_SUBSET; @@ -674,9 +718,11 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, } // compute map for select items output - if (OB_SUCC(ret) && QueryRelation::QUERY_UNCOMPARABLE != relation) { + if (OB_SUCC(ret) && need_check_select_items + && QueryRelation::QUERY_UNCOMPARABLE != relation) { ObSEArray first_exprs; ObSEArray second_exprs; + QueryRelation this_relation; if (OB_FAIL(first_sel->get_select_exprs(first_exprs))) { LOG_WARN("failed to get select exprs", K(ret)); } else if (OB_FAIL(second_sel->get_select_exprs(second_exprs))) { @@ -687,10 +733,11 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, second_exprs, map_info, map_info.select_item_map_, - match_count, + this_relation, + is_in_same_stmt, is_strict_select_list))) { LOG_WARN("failed to compute output expr map", K(ret)); - } else if (match_count == first_exprs.count() && match_count == second_exprs.count()) { + } else if (QueryRelation::QUERY_EQUAL == this_relation) { map_info.is_select_item_equal_ = true; LOG_TRACE("succeed to check select item map", K(relation), K(map_info)); } else { @@ -703,11 +750,13 @@ int ObStmtComparer::check_stmt_containment(const ObDMLStmt *first, int ObStmtComparer::compute_from_items_map(const ObDMLStmt *first, const ObDMLStmt *second, + bool is_in_same_stmt, ObStmtMapInfo &map_info, - int64_t &match_count) + QueryRelation &relation) { int ret = OB_SUCCESS; - match_count = 0; + int match_count = 0; + relation = QueryRelation::QUERY_UNCOMPARABLE; if (OB_ISNULL(first) || OB_ISNULL(second)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(first), K(second), K(ret)); @@ -734,6 +783,7 @@ int ObStmtComparer::compute_from_items_map(const ObDMLStmt *first, first_from, second, second_from, + is_in_same_stmt, map_info, is_match))) { LOG_WARN("failed to check the from item same", K(ret)); @@ -748,6 +798,14 @@ int ObStmtComparer::compute_from_items_map(const ObDMLStmt *first, } } } + if (OB_FAIL(ret)) { + } else if (match_count != first->get_from_item_size()) { + relation = QueryRelation::QUERY_UNCOMPARABLE; + } else { + relation = QueryRelation::QUERY_EQUAL; + map_info.is_table_equal_ = true; + map_info.is_from_equal_ = true; + } return ret; } @@ -757,13 +815,16 @@ int ObStmtComparer::compute_conditions_map(const ObDMLStmt *first, const ObIArray &second_exprs, ObStmtMapInfo &map_info, ObIArray &condition_map, - int64_t &match_count, - bool is_same_by_order) + QueryRelation &relation, + bool is_in_same_cond, + bool is_same_by_order, + bool need_check_second_range) { int ret = OB_SUCCESS; ObSqlBitSet<> matched_items; - ObStmtCompareContext context(first, second, map_info, &first->get_query_ctx()->calculable_items_); - match_count = 0; + ObStmtCompareContext context(first, second, map_info, &first->get_query_ctx()->calculable_items_, false, is_in_same_cond); + int match_count = 0; + relation = QueryRelation::QUERY_UNCOMPARABLE; if (OB_ISNULL(first) || OB_ISNULL(second) || OB_ISNULL(first->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(first), K(second), K(ret)); @@ -787,6 +848,8 @@ int ObStmtComparer::compute_conditions_map(const ObDMLStmt *first, // do nothing } else if (OB_FAIL(append(map_info.equal_param_map_, context.equal_param_info_))) { LOG_WARN("failed to append exprs", K(ret)); + } else if (OB_FAIL(append(map_info.const_param_map_, context.const_param_info_))) { + LOG_WARN("failed to append exprs", K(ret)); } else if (OB_FAIL(matched_items.add_member(j))) { LOG_WARN("failed to add member", K(ret)); } else { @@ -806,6 +869,8 @@ int ObStmtComparer::compute_conditions_map(const ObDMLStmt *first, // do nothing } else if (OB_FAIL(append(map_info.equal_param_map_, context.equal_param_info_))) { LOG_WARN("failed to append exprs", K(ret)); + } else if (OB_FAIL(append(map_info.const_param_map_, context.const_param_info_))) { + LOG_WARN("failed to append exprs", K(ret)); } else if (OB_FAIL(matched_items.add_member(i))) { LOG_WARN("failed to add member", K(ret)); } else { @@ -816,6 +881,169 @@ int ObStmtComparer::compute_conditions_map(const ObDMLStmt *first, } } } + + // handle unmatched conditions + ObSEArray first_unmatched_conds; + ObSEArray second_unmatched_conds; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(compute_unmatched_item(condition_map, + first_exprs.count(), + second_exprs.count(), + first_unmatched_conds, + second_unmatched_conds))) { + LOG_WARN("failed to compute unmatched conditions", K(ret)); + } + + if (OB_FAIL(ret)) { + } else if (first_unmatched_conds.count() == 0 + && second_unmatched_conds.count() == 0) { + relation = QUERY_EQUAL; + } else if (second_unmatched_conds.count() == 0) { + relation = QUERY_LEFT_SUBSET; + } else if (first_unmatched_conds.count() == 0) { + relation = QUERY_RIGHT_SUBSET; + } else { + relation = QUERY_UNCOMPARABLE; + } + return ret; +} + +// try to compute target expr using source expr +int ObStmtComparer::compute_new_expr(const ObIArray &target_exprs, + const ObDMLStmt *target_stmt, + const ObIArray &source_exprs, + const ObDMLStmt *source_stmt, + ObStmtMapInfo &map_info, + ObRawExprCopier &expr_copier, + ObIArray &compute_exprs, + bool &is_all_computable) +{ + int ret = OB_SUCCESS; + is_all_computable = true; + if (OB_ISNULL(source_stmt) || OB_ISNULL(target_stmt) || OB_ISNULL(target_stmt->get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected NULL", K(ret), K(source_stmt), K(target_stmt)); + } else if (OB_FAIL(compute_exprs.prepare_allocate(target_exprs.count()))) { + LOG_WARN("failed to allocate array", K(ret)); + } else { + ObStmtCompareContext context(target_stmt, source_stmt, map_info, + &target_stmt->get_query_ctx()->calculable_items_, false , false); + for (int64_t i = 0; OB_SUCC(ret) && i < target_exprs.count(); ++i) { + bool is_match = false; + ObRawExpr *new_expr = NULL; + if (OB_FAIL(inner_compute_expr(target_exprs.at(i), source_exprs, context, expr_copier, is_match))) { + LOG_WARN("failed to compute expr", K(ret), K(i), KPC(target_exprs.at(i))); + } else if (!is_match) { + is_all_computable = false; + } else if (OB_FAIL(expr_copier.copy_on_replace(target_exprs.at(i), new_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(target_exprs.at(i))); + } else if (OB_FAIL(append(map_info.equal_param_map_, context.equal_param_info_))) { + LOG_WARN("failed to append equal param info", K(ret)); + } else if (OB_FAIL(append(map_info.const_param_map_, context.const_param_info_))) { + LOG_WARN("failed to append exprs", K(ret)); + } else { + compute_exprs.at(i) = new_expr; + } + } + } + return ret; +} + +int ObStmtComparer::inner_compute_expr(const ObRawExpr *target_expr, + const ObIArray &source_exprs, + ObStmtCompareContext &context, + ObRawExprCopier &expr_copier, + bool &is_match) +{ + int ret = OB_SUCCESS; + is_match = false; + if (OB_ISNULL(target_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected NULL", K(ret), K(target_expr)); + } else if (target_expr->is_const_expr()) { + // Do not need match + is_match = true; + } + // Try to match the complete expr + for (int64_t i = 0; OB_SUCC(ret) && !is_match && i < source_exprs.count(); ++i) { + ObStmtCompareContext new_ctx(context.inner_, + context.outer_, + context.map_info_, + context.calculable_items_, + context.need_check_deterministic_, + context.is_in_same_stmt_); + if (OB_FAIL(is_same_condition(target_expr, source_exprs.at(i), new_ctx, is_match))) { + LOG_WARN("failed to check is condition equal", K(ret)); + } else if (!is_match) { + // do nothing + } else if (OB_FAIL(expr_copier.add_replaced_expr(target_expr, source_exprs.at(i)))) { + LOG_WARN("failed to add replaceed expr", K(ret), KPC(target_expr), KPC(source_exprs.at(i))); + } else if (OB_FAIL(append(context.equal_param_info_, new_ctx.equal_param_info_))) { + LOG_WARN("failed to append expr", K((ret))); + } else if (OB_FAIL(append(context.const_param_info_, new_ctx.const_param_info_))) { + LOG_WARN("failed to append exprs", K(ret)); + } + } + // Try to match each param expr + if (OB_SUCC(ret) && !is_match + && (ObRawExpr::EXPR_OPERATOR == target_expr->get_expr_class() + || ObRawExpr::EXPR_AGGR == target_expr->get_expr_class() + || ObRawExpr::EXPR_SYS_FUNC == target_expr->get_expr_class() + || ObRawExpr::EXPR_WINDOW == target_expr->get_expr_class() + || ObRawExpr::EXPR_UDF == target_expr->get_expr_class())) { + bool sub_match = true; + for (int64_t i = 0; OB_SUCC(ret) && sub_match && i < target_expr->get_param_count(); ++i) { + if (OB_FAIL(SMART_CALL(inner_compute_expr(target_expr->get_param_expr(i), + source_exprs, + context, + expr_copier, + sub_match)))) { + LOG_WARN("failed to check param expr", K(ret)); + } + } + if (OB_SUCC(ret) && sub_match) { + is_match = true; + } + } + return ret; +} + +int ObStmtComparer::compute_unmatched_item(const ObIArray &item_map, + int first_size, + int second_size, + ObIArray &first_unmatched_items, + ObIArray &second_unmatched_items) +{ + int ret = OB_SUCCESS; + ObSqlBitSet<> first_matched_set; + ObSqlBitSet<> second_matched_set; + for (int64_t i = 0; OB_SUCC(ret) && i < first_size; ++i) { + if (OB_UNLIKELY(i >= item_map.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("item map size overflow", K(ret), K(i), K(first_size), K(item_map)); + } else if (item_map.at(i) != OB_INVALID_ID) { + if (OB_UNLIKELY(item_map.at(i) < 0 || item_map.at(i) >= second_size)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected map value", K(ret), K(i), K(item_map.at(i))); + } else if (OB_FAIL(first_matched_set.add_member(i))) { + LOG_WARN("failed to add member", K(ret), K(i)); + } else if (OB_FAIL(second_matched_set.add_member(item_map.at(i)))) { + LOG_WARN("failed to add member", K(ret), K(item_map.at(i))); + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < first_size; ++i) { + if (!first_matched_set.has_member(i) + && (OB_FAIL(first_unmatched_items.push_back(i)))) { + LOG_WARN("failed to push back", K(ret), K(i)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < second_size; ++i) { + if (!second_matched_set.has_member(i) + && (OB_FAIL(second_unmatched_items.push_back(i)))) { + LOG_WARN("failed to push back", K(ret), K(i)); + } + } return ret; } @@ -847,6 +1075,8 @@ int ObStmtComparer::compute_orderby_map(const ObDMLStmt *first, first_match_all = false; } else if (OB_FAIL(append(map_info.equal_param_map_, context.equal_param_info_))) { LOG_WARN("failed to append exprs", K(ret)); + } else if (OB_FAIL(append(map_info.const_param_map_, context.const_param_info_))) { + LOG_WARN("failed to append exprs", K(ret)); } else { match_count++; } @@ -863,11 +1093,13 @@ int ObStmtComparer::is_same_condition(const ObRawExpr *left, int ret = OB_SUCCESS; is_same = false; context.equal_param_info_.reset(); + context.const_param_info_.reset(); if (OB_ISNULL(left) || OB_ISNULL(right)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr is null", K(ret)); } else if (!(is_same = left->same_as(*right, &context))) { context.equal_param_info_.reset(); + context.const_param_info_.reset(); if (!IS_COMMON_COMPARISON_OP(left->get_expr_type()) || get_opposite_compare_type(left->get_expr_type()) != right->get_expr_type()) { // do nothing @@ -890,6 +1122,7 @@ int ObStmtComparer::is_same_condition(const ObRawExpr *left, int ObStmtComparer::compute_semi_infos_map(const ObDMLStmt *first, const ObDMLStmt *second, + bool is_in_same_stmt, ObStmtMapInfo &map_info, int64_t &match_count) { @@ -914,6 +1147,7 @@ int ObStmtComparer::compute_semi_infos_map(const ObDMLStmt *first, first_semi_infos.at(i), second, second_semi_infos.at(j), + is_in_same_stmt, map_info, is_match))) { LOG_WARN("failed to check is condition equal", K(ret)); @@ -935,6 +1169,7 @@ int ObStmtComparer::is_same_semi_info(const ObDMLStmt *first, const SemiInfo *first_semi_info, const ObDMLStmt *second, const SemiInfo *second_semi_info, + bool is_in_same_stmt, ObStmtMapInfo &map_info, bool &is_same) { @@ -973,6 +1208,7 @@ int ObStmtComparer::is_same_semi_info(const ObDMLStmt *first, first_table, second, second_table, + is_in_same_stmt, map_info, relation))) { LOG_WARN("failed to compare table item", K(ret)); @@ -985,16 +1221,17 @@ int ObStmtComparer::is_same_semi_info(const ObDMLStmt *first, //check semi condition if (OB_SUCC(ret) && is_same) { ObSEArray condition_map; + QueryRelation relation = QueryRelation::QUERY_UNCOMPARABLE; if (OB_FAIL(compute_conditions_map(first, second, first_semi_info->semi_conditions_, second_semi_info->semi_conditions_, map_info, condition_map, - match_count))) { + relation, + is_in_same_stmt))) { LOG_WARN("failed to compute conditions map", K(ret)); - } else if (match_count == first_semi_info->semi_conditions_.count() && - match_count == second_semi_info->semi_conditions_.count()) { + } else if (relation == QueryRelation::QUERY_EQUAL) { is_same = true; } else { is_same = false; @@ -1089,10 +1326,11 @@ int ObStmtComparer::compare_basic_table_item(const ObDMLStmt *first, return ret; } -int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, +int ObStmtComparer::compare_joined_table_item(const ObDMLStmt *first, const TableItem *first_table, const ObDMLStmt *second, const TableItem *second_table, + bool is_in_same_stmt, ObStmtMapInfo &map_info, QueryRelation &relation) { @@ -1100,6 +1338,7 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, relation = QueryRelation::QUERY_UNCOMPARABLE; QueryRelation left_relation = QueryRelation::QUERY_UNCOMPARABLE; QueryRelation right_relation = QueryRelation::QUERY_UNCOMPARABLE; + QueryRelation cmp_relation = QueryRelation::QUERY_UNCOMPARABLE; if (OB_ISNULL(first) || OB_ISNULL(first_table) || OB_ISNULL(second) || OB_ISNULL(second_table)) { ret = OB_INVALID_ARGUMENT; @@ -1117,6 +1356,7 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, first_joined_table->left_table_, second, second_joined_table->left_table_, + is_in_same_stmt, map_info, left_relation)))) { LOG_WARN("compare joined table failed", K(ret)); @@ -1124,6 +1364,7 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, first_joined_table->right_table_, second, second_joined_table->right_table_, + is_in_same_stmt, map_info, right_relation)))) { LOG_WARN("compare joined table failed", K(ret)); @@ -1136,6 +1377,7 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, first_joined_table->left_table_, second, second_joined_table->right_table_, + is_in_same_stmt, map_info, left_relation)))) { LOG_WARN("compare joined table failed", K(ret)); @@ -1143,6 +1385,7 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, first_joined_table->right_table_, second, second_joined_table->left_table_, + is_in_same_stmt, map_info, right_relation)))) { LOG_WARN("compare joined table failed", K(ret)); @@ -1159,9 +1402,10 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, second_joined_table->join_conditions_, map_info, condition_map, - match_count))) { + cmp_relation, + is_in_same_stmt))) { LOG_WARN("failed to compute conditions map", K(ret)); - } else if (match_count != first_count || match_count != second_count) { + } else if (cmp_relation != QueryRelation::QUERY_EQUAL) { //on condition不相同 } else { relation = QueryRelation::QUERY_EQUAL; @@ -1170,6 +1414,7 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, first_table, second, second_table, + is_in_same_stmt, map_info, relation))) { LOG_WARN("failed to compare table item", K(ret)); @@ -1178,11 +1423,12 @@ int ObStmtComparer::compare_joined_table_item (const ObDMLStmt *first, } int ObStmtComparer::compare_table_item(const ObDMLStmt *first, - const TableItem *first_table, - const ObDMLStmt *second, - const TableItem *second_table, - ObStmtMapInfo &map_info, - QueryRelation &relation) + const TableItem *first_table, + const ObDMLStmt *second, + const TableItem *second_table, + bool is_in_same_stmt, + ObStmtMapInfo &map_info, + QueryRelation &relation) { int ret = OB_SUCCESS; relation = QueryRelation::QUERY_UNCOMPARABLE; @@ -1244,7 +1490,10 @@ int ObStmtComparer::compare_table_item(const ObDMLStmt *first, } else if (OB_FAIL(SMART_CALL(check_stmt_containment(first_table->ref_query_, second_table->ref_query_, ref_query_map_info, - relation)))) { + relation, + false, + true, + is_in_same_stmt)))) { LOG_WARN("check stmt containment failed", K(ret)); } else if (OB_FAIL(map_info.view_select_item_map_.at(first_table_index - 1).assign(ref_query_map_info.select_item_map_))) { LOG_WARN("failed to assign select item map", K(ret)); @@ -1264,6 +1513,7 @@ int ObStmtComparer::compare_table_item(const ObDMLStmt *first, first_table, second, second_table, + is_in_same_stmt, map_info, relation))) { LOG_WARN("failed to compare joined table item", K(ret)); @@ -1293,7 +1543,8 @@ int ObStmtComparer::compare_table_item(const ObDMLStmt *first, int ObStmtComparer::compare_set_stmt(const ObSelectStmt *first, const ObSelectStmt *second, ObStmtMapInfo &map_info, - QueryRelation &relation) + QueryRelation &relation, + bool is_in_same_stmt) { int ret = OB_SUCCESS; relation = QueryRelation::QUERY_UNCOMPARABLE; @@ -1315,7 +1566,10 @@ int ObStmtComparer::compare_set_stmt(const ObSelectStmt *first, if (OB_FAIL(SMART_CALL(check_stmt_containment(first->get_set_query(i), second->get_set_query(i), ref_query_map_info, - set_query_relation)))) { + set_query_relation, + false, + true, + is_in_same_stmt)))) { LOG_WARN("check stmt containment failed", K(ret)); } else if (QueryRelation::QUERY_EQUAL == set_query_relation && ref_query_map_info.is_select_item_equal_) { if (OB_FAIL(map_info.view_select_item_map_.push_back(ref_query_map_info.select_item_map_))) { @@ -1450,6 +1704,8 @@ int ObStmtComparer::compare_values_table_item(const ObDMLStmt *first, // do nothing } else if (OB_FAIL(append(map_info.equal_param_map_, context.equal_param_info_))) { LOG_WARN("failed to append exprs", K(ret)); + } else if (OB_FAIL(append(map_info.const_param_map_, context.const_param_info_))) { + LOG_WARN("failed to append exprs", K(ret)); } else { ++match_count; } diff --git a/src/sql/rewrite/ob_stmt_comparer.h b/src/sql/rewrite/ob_stmt_comparer.h index 51fd19bee..98910ef55 100644 --- a/src/sql/rewrite/ob_stmt_comparer.h +++ b/src/sql/rewrite/ob_stmt_comparer.h @@ -36,6 +36,8 @@ enum QueryRelation }; struct ObStmtMapInfo { common::ObSEArray, 4> view_select_item_map_; + common::ObSEArray expr_cons_map_; + common::ObSEArray const_param_map_; common::ObSEArray equal_param_map_; common::ObSEArray table_map_; common::ObSEArray from_map_; @@ -54,7 +56,8 @@ enum QueryRelation bool is_select_item_equal_; bool is_distinct_equal_; bool is_qualify_filter_equal_; - + bool left_can_be_replaced_; // used for mv rewrite + //如果from item是generated table，需要记录ref query的select item map关系 //如果是set stmt，每个set query对应的映射关系也记录在view_select_item_map_ ObStmtMapInfo() @@ -67,7 +70,8 @@ enum QueryRelation is_order_equal_(false), is_select_item_equal_(false), is_distinct_equal_(false), - is_qualify_filter_equal_(false) + is_qualify_filter_equal_(false), + left_can_be_replaced_(true) {} void reset(); @@ -83,7 +87,8 @@ enum QueryRelation K_(equal_param_map), K_(view_select_item_map), K_(is_order_equal), - K_(is_distinct_equal)); + K_(is_distinct_equal), + K_(left_can_be_replaced)); }; struct StmtCompareHelper { @@ -119,7 +124,8 @@ struct ObStmtCompareContext : ObExprEqualCheckContext inner_(NULL), outer_(NULL), map_info_(), - equal_param_info_() + equal_param_info_(), + is_in_same_stmt_(true) { init_override_params(); } @@ -129,19 +135,22 @@ struct ObStmtCompareContext : ObExprEqualCheckContext inner_(NULL), outer_(NULL), map_info_(), - equal_param_info_() + equal_param_info_(), + is_in_same_stmt_(true) { init_override_params(); } // for common expression extraction ObStmtCompareContext(const ObIArray *calculable_items, - bool need_check_deterministic = false) : + bool need_check_deterministic = false, + bool is_in_same_stmt = true) : ObExprEqualCheckContext(need_check_deterministic), calculable_items_(calculable_items), inner_(NULL), outer_(NULL), map_info_(), - equal_param_info_() + equal_param_info_(), + is_in_same_stmt_(is_in_same_stmt) { init_override_params(); } @@ -149,13 +158,15 @@ struct ObStmtCompareContext : ObExprEqualCheckContext const ObDMLStmt *outer, const ObStmtMapInfo &map_info, const ObIArray *calculable_items, - bool need_check_deterministic = false) : + bool need_check_deterministic = false, + bool is_in_same_stmt = true) : ObExprEqualCheckContext(need_check_deterministic), calculable_items_(calculable_items), inner_(inner), outer_(outer), map_info_(map_info), - equal_param_info_() + equal_param_info_(), + is_in_same_stmt_(is_in_same_stmt) { init_override_params(); } @@ -202,6 +213,9 @@ struct ObStmtCompareContext : ObExprEqualCheckContext const ObDMLStmt *outer_; ObStmtMapInfo map_info_; common::ObSEArray equal_param_info_; + common::ObSEArray expr_cons_info_; + common::ObSEArray const_param_info_; + bool is_in_same_stmt_; // only if the two stmts are in the same parent stmt, can we compare table id and column id directly }; class ObStmtComparer @@ -226,7 +240,9 @@ public: const ObDMLStmt *second, ObStmtMapInfo &map_info, QueryRelation &relation, - bool is_strict_select_list = false); + bool is_strict_select_list = false, + bool need_check_select_items = true, + bool is_in_same_stmt = true); static int compute_conditions_map(const ObDMLStmt *first, const ObDMLStmt *second, @@ -234,8 +250,10 @@ public: const ObIArray &second_exprs, ObStmtMapInfo &map_info, ObIArray &condition_map, - int64_t &match_count, - bool is_same_by_order = false); + QueryRelation &relation, + bool is_in_same_cond = true, + bool is_same_by_order = false, + bool need_check_second_range = false); static int compute_orderby_map(const ObDMLStmt *first, const ObDMLStmt *second, @@ -246,13 +264,15 @@ public: static int compute_from_items_map(const ObDMLStmt *first, const ObDMLStmt *second, + bool is_in_same_stmt, ObStmtMapInfo &map_info, - int64_t &match_count); + QueryRelation &relation); static int is_same_from(const ObDMLStmt *first, const FromItem &first_from, const ObDMLStmt *second, const FromItem &second_from, + bool is_in_same_stmt, ObStmtMapInfo &map_info, bool &is_same); @@ -263,6 +283,7 @@ public: static int compute_semi_infos_map(const ObDMLStmt *first, const ObDMLStmt *second, + bool is_in_same_stmt, ObStmtMapInfo &map_info, int64_t &match_count); @@ -270,6 +291,7 @@ public: const SemiInfo *first_semi_info, const ObDMLStmt *second, const SemiInfo *second_semi_info, + bool is_in_same_stmt, ObStmtMapInfo &map_info, bool &is_same); @@ -281,8 +303,29 @@ public: ObIArray &table_map, int64_t &match_count); + static int compute_unmatched_item(const ObIArray &item_map, + int first_size, + int second_size, + ObIArray &first_unmatched_items, + ObIArray &second_unmatched_items); + + static int compute_new_expr(const ObIArray &target_exprs, + const ObDMLStmt *target_stmt, + const ObIArray &source_exprs, + const ObDMLStmt *source_stmt, + ObStmtMapInfo &map_info, + ObRawExprCopier &expr_copier, + ObIArray &compute_exprs, + bool &is_all_computable); + + static int inner_compute_expr(const ObRawExpr *target_expr, + const ObIArray &source_exprs, + ObStmtCompareContext &context, + ObRawExprCopier &expr_copier, + bool &is_match); + /** - * @brief compare_basic_table_item + * @brief compare_basic_table_item * 如果两张表partition hint分区包含的关系， * 如果两张表不同，则不可比较 * 如果两张表相同且没有partition hint，则相等 @@ -303,6 +346,7 @@ public: const TableItem *first_table, const ObDMLStmt *second, const TableItem *second_table, + bool is_in_same_stmt, ObStmtMapInfo &map_info, QueryRelation &relation); @@ -314,13 +358,15 @@ public: const TableItem *first_table, const ObDMLStmt *second, const TableItem *second_table, + bool is_in_same_stmt, ObStmtMapInfo &map_info, QueryRelation &relation); static int compare_set_stmt(const ObSelectStmt *first, const ObSelectStmt *second, ObStmtMapInfo &map_info, - QueryRelation &relation); + QueryRelation &relation, + bool is_in_same_stmt = true); static int compare_values_table_item(const ObDMLStmt *first, const TableItem *first_table, diff --git a/src/sql/rewrite/ob_transform_mv_rewrite.cpp b/src/sql/rewrite/ob_transform_mv_rewrite.cpp new file mode 100644 index 000000000..ea9831526 --- /dev/null +++ b/src/sql/rewrite/ob_transform_mv_rewrite.cpp @@ -0,0 +1,1306 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_REWRITE +#include "sql/rewrite/ob_transform_mv_rewrite.h" +#include "sql/rewrite/ob_transform_pre_process.h" +#include "sql/resolver/dml/ob_select_resolver.h" +#include "sql/resolver/expr/ob_raw_expr_wrap_enum_set.h" +#include "lib/mysqlclient/ob_mysql_result.h" +#include "share/schema/ob_table_schema.h" + +namespace oceanbase +{ +namespace sql +{ + +int ObTransformMVRewrite::transform_one_stmt(common::ObIArray &parent_stmts, + ObDMLStmt *&stmt, + bool &trans_happened) +{ + int ret = OB_SUCCESS; + trans_happened = false; + const ObDMLStmt *root_stmt = parent_stmts.empty() ? stmt : parent_stmts.at(0).stmt_; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(stmt)); + } else if (!stmt->is_select_stmt()) { + // do nothing + } else if (OB_FAIL(prepare_mv_info(root_stmt, stmt))) { + LOG_WARN("failed to prepate mv info", K(ret)); + } else if (OB_FAIL(do_transform(stmt, trans_happened))) { + LOG_WARN("failed to do transform", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::need_transform(const common::ObIArray &parent_stmts, + const int64_t current_level, + const ObDMLStmt &stmt, + bool &need_trans) +{ + int ret = OB_SUCCESS; + int64_t query_rewrite_enabled = QueryRewriteEnabledType::REWRITE_ENABLED_FALSE; + bool has_mv = false; + bool hint_rewrite = false; + bool hint_no_rewrite = false; + bool is_stmt_valid = false; + uint64_t data_version; + need_trans = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexprcted null", K(ret), K(ctx_)); + } else if (OB_FAIL(ObTransformRule::need_transform(parent_stmts, + current_level, + stmt, + need_trans))) { + LOG_WARN("failed to check need transformation", K(ret)); + } else if (!need_trans) { + // do nothing + } else if (ctx_->iteration_level_ > 0) { + need_trans = false; + OPT_TRACE("only do mv rewrite in the first iteration"); + } else if (stmt.is_select_stmt() && static_cast(&stmt)->is_expanded_mview()) { + need_trans = false; + } else if (ctx_->session_info_->is_inner() || !ctx_->session_info_->is_user_session()) { + need_trans = false; + OPT_TRACE("not a user SQL, skip mv rewrite"); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(ctx_->session_info_->get_effective_tenant_id(), data_version))) { + LOG_WARN("failed to get data version", K(ret), K(ctx_->session_info_->get_effective_tenant_id())); + } else if (OB_UNLIKELY(data_version < DATA_VERSION_4_3_1_0)) { + // data version lower than 4.3.1 does not have the inner table used to get mv list + need_trans = false; + OPT_TRACE("min data version is lower than 4.3.1, skip mv rewrite"); + } else if (OB_FAIL(check_hint_valid(stmt, hint_rewrite, hint_no_rewrite))) { + LOG_WARN("failed to check mv rewrite hint", K(ret)); + } else if (hint_rewrite) { + need_trans = true; + OPT_TRACE("hint force mv rewrite"); + } else if (hint_no_rewrite) { + need_trans = false; + OPT_TRACE("hint reject mv rewrite"); + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_enabled(query_rewrite_enabled))) { + LOG_WARN("failed to get query rewrite enabled", K(ret)); + } else if (QueryRewriteEnabledType::REWRITE_ENABLED_FALSE == query_rewrite_enabled) { + need_trans = false; + OPT_TRACE("system variable reject mv rewrite"); + } else if (OB_FAIL(check_table_has_mv(stmt, has_mv))) { + LOG_WARN("failed to check table has mv", K(ret)); + } else if (!has_mv) { + need_trans = false; + OPT_TRACE("table does not have mv, no need to rewrite"); + } else if (OB_FAIL(check_basic_validity(stmt, is_stmt_valid))) { + LOG_WARN("failed to check basic validity", K(ret)); + } else if (!is_stmt_valid) { + need_trans = false; + OPT_TRACE("stmt can not do mv rewrite"); + } else { + need_trans = true; + } + return ret; +} + +int ObTransformMVRewrite::check_hint_valid(const ObDMLStmt &stmt, + bool &force_rewrite, + bool &force_no_rewrite) +{ + int ret = OB_SUCCESS; + force_rewrite = false; + force_no_rewrite = false; + const ObQueryHint *query_hint = NULL; + const ObMVRewriteHint *myhint = static_cast(get_hint(stmt.get_stmt_hint())); + if (OB_ISNULL(query_hint = stmt.get_stmt_hint().query_hint_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query hint is null", K(ret), K(stmt)); + } else { + force_rewrite = NULL != myhint && myhint->is_enable_hint(); + force_no_rewrite = !force_rewrite && query_hint->has_outline_data(); + // force_no_rewrite does not need to handle (NULL != myhint && myhint->is_disable_hint()), + // because it has been handled in ObTransformRule::check_hint_status + } + return ret; +} + +int ObTransformMVRewrite::check_table_has_mv(const ObDMLStmt &stmt, + bool &has_mv) +{ + int ret = OB_SUCCESS; + has_mv = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } + for (int64_t i = 0; OB_SUCC(ret) && !has_mv && i < stmt.get_table_size(); ++i) { + const TableItem *table = NULL; + const ObTableSchema *table_schema = NULL; + if (OB_ISNULL(table = stmt.get_table_item(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret), K(i), K(stmt.get_table_size())); + } else if (!table->is_basic_table()) { + // do nothing + } else if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), + table->ref_id_, + table_schema))) { + LOG_WARN("failed to get table schema", K(ret), K(i), K(table->ref_id_)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret), K(table->ref_id_)); + } else if (ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV == + ObTableMode::get_table_referenced_by_mv_flag(table_schema->get_table_mode())) { + has_mv = true; + } + } + return ret; +} + +int ObTransformMVRewrite::check_basic_validity(const ObDMLStmt &stmt, bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray seq_exprs; + is_valid = true; + if (!stmt.is_select_stmt()) { + OPT_TRACE("stmt can not do mv rewrite, not a select stmt"); + is_valid = false; + } else if (stmt.is_set_stmt()) { + OPT_TRACE("stmt can not do mv rewrite, is a set stmt"); + is_valid = false; + } else if (stmt.is_hierarchical_query()) { + OPT_TRACE("stmt can not do mv rewrite, is a hierarchical query"); + is_valid = false; + } else if (stmt.is_contains_assignment()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains assignment"); + is_valid = false; + } else if (stmt.has_for_update()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains for update"); + is_valid = false; + } else if (stmt.has_ora_rowscn()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains ora rowscn"); + is_valid = false; + } else if (stmt.is_unpivot_select()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains unpivot"); + is_valid = false; + } else if (!stmt.get_pseudo_column_like_exprs().empty()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains pseudo column"); + is_valid = false; + } else if (static_cast(&stmt)->has_rollup()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains roll up"); + is_valid = false; + } else if (static_cast(&stmt)->has_select_into()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains select into"); + is_valid = false; + } else if (OB_FAIL(stmt.get_sequence_exprs(seq_exprs))) { + LOG_WARN("failed to get sequence exprs", K(ret)); + } else if (!seq_exprs.empty()) { + OPT_TRACE("stmt can not do mv rewrite, is a contains sequence"); + is_valid = false; + } + return ret; +} + +int ObTransformMVRewrite::construct_transform_hint(ObDMLStmt &stmt, void *trans_params) +{ + int ret = OB_SUCCESS; + const MvInfo *mv_info = NULL; + ObMVRewriteHint *hint = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) + || OB_ISNULL(mv_info = static_cast(trans_params)) + || OB_ISNULL(mv_info->mv_schema_) || OB_ISNULL(mv_info->db_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(trans_params)); + } else if (OB_FAIL(ObQueryHint::create_hint(ctx_->allocator_, T_MV_REWRITE, hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(ctx_->outline_trans_hints_.push_back(hint))) { + LOG_WARN("failed to push back hint", K(ret)); + } else if (OB_FAIL(ctx_->add_used_trans_hint(get_hint(stmt.get_stmt_hint())))) { + LOG_WARN("failed to add used trans hint", K(ret)); + } else if (OB_FAIL(hint->get_mv_list().push_back( + ObTableInHint(NULL, mv_info->db_schema_->get_database_name(), mv_info->mv_schema_->get_table_name())))) { + LOG_WARN("failed to push back hint table", K(ret)); + } else { + hint->set_qb_name(ctx_->src_qb_name_); + } + return ret; +} + +int ObTransformMVRewrite::prepare_mv_info(const ObDMLStmt *root_stmt, + const ObDMLStmt *stmt) +{ + int ret = OB_SUCCESS; + ObSEArray mv_list; + if (is_mv_info_generated_) { + // do nothing + } else if (OB_FAIL(get_mv_list(root_stmt, mv_list))) { + LOG_WARN("failed to get mv list", K(ret)); + } else if (OB_FAIL(generate_mv_info(stmt, mv_list))) { + LOG_WARN("failed to generate mv info", K(ret)); + } else { + is_mv_info_generated_ = true; + OPT_TRACE("use", mv_infos_.count(), "materialized view(s) to perform rewrite"); + } + return ret; +} + +int ObTransformMVRewrite::get_mv_list(const ObDMLStmt *root_stmt, + ObIArray &mv_list) +{ + int ret = OB_SUCCESS; + ObMySQLProxy *sql_proxy = NULL; + ObSqlString sql; + sqlclient::ObMySQLResult *mysql_result = NULL; + ObSqlString table_ids; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->exec_ctx_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(get_base_table_id_string(root_stmt, table_ids))) { + LOG_WARN("failed to get base table id string", K(ret)); + } else if (OB_ISNULL(sql_proxy = ctx_->exec_ctx_->get_sql_proxy())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", K(ret)); + } else if (OB_FAIL(sql.assign_fmt("SELECT DISTINCT MVIEW_ID FROM `%s`.`%s` WHERE TENANT_ID = 0 AND P_OBJ IN (%.*s)", + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_DEP_TNAME, + static_cast(table_ids.length()), table_ids.ptr()))) { + LOG_WARN("failed to assign sql", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + if (OB_FAIL(sql_proxy->read(res, ctx_->session_info_->get_effective_tenant_id(), sql.ptr()))) { + LOG_WARN("execute sql failed", K(ret), K(sql)); + } else if (OB_ISNULL(mysql_result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get result", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(mysql_result->next())) { + int64_t mv_id = OB_INVALID_ID; + if (OB_FAIL(mysql_result->get_int(0L, mv_id))) { + LOG_WARN("failed to get mv id", K(ret)); + } else if (OB_FAIL(mv_list.push_back(mv_id))) { + LOG_WARN("failed to push back mv id", K(ret), K(mv_id)); + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + OPT_TRACE("get", mv_list.count(), "materialized view(s)"); + } else if (OB_FAIL(ret)) { + LOG_WARN("failed to get inner sql result", K(ret)); + } + } + } + } + return ret; +} + +int ObTransformMVRewrite::get_base_table_id_string(const ObDMLStmt *stmt, + ObSqlString &table_ids) +{ + int ret = OB_SUCCESS; + ObSEArray table_id_list; + ObSqlBitSet<> visited_id; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(stmt)); + } else if (OB_FAIL(get_all_base_table_id(stmt, table_id_list))) { + LOG_WARN("failed to get table ids", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < table_id_list.count(); ++i) { + if (visited_id.has_member(table_id_list.at(i))) { + // do nothing + } else if (OB_FAIL(visited_id.add_member(table_id_list.at(i)))) { + LOG_WARN("failed to add member", K(ret)); + } else if (i > 0 && OB_FAIL(table_ids.append(","))) { + LOG_WARN("failed to append comma", K(ret)); + } else if (OB_FAIL(table_ids.append(to_cstring(table_id_list.at(i))))) { + LOG_WARN("failed to append table id", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewrite::get_all_base_table_id(const ObDMLStmt *stmt, + ObIArray &table_ids) +{ + int ret = OB_SUCCESS; + ObSEArray child_stmts; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(stmt)); + } else if (OB_FAIL(stmt->get_child_stmts(child_stmts))) { + LOG_WARN("failed to get child stmts", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); ++i) { + const TableItem *table = NULL; + if (OB_ISNULL(table = stmt->get_table_item(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(i)); + } else if (!table->is_basic_table()) { + // do nothing + } else if (OB_FAIL(table_ids.push_back(table->ref_id_))) { + LOG_WARN("failed to push back table id", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < child_stmts.count(); ++i) { + if (OB_FAIL(SMART_CALL(get_all_base_table_id(child_stmts.at(i), table_ids)))) { + LOG_WARN("failed to get all base table id", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewrite::generate_mv_info(const ObDMLStmt *stmt, + ObIArray &mv_list) +{ + int ret = OB_SUCCESS; + int64_t query_rewrite_integrity = QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) + || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(stmt)); + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_integrity(query_rewrite_integrity))) { + LOG_WARN("failed to get query rewrite integrity", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < mv_list.count(); ++i) { + const ObTableSchema *mv_schema = NULL; + const ObDatabaseSchema *db_schema = NULL; + bool is_valid = true; + if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), + mv_list.at(i), + mv_schema))) { + LOG_WARN("failed to get mv schema", K(ret), K(mv_list.at(i))); + } else if (OB_ISNULL(mv_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv schema is null", K(ret), K(mv_list.at(i))); + } else if (OB_FAIL(ctx_->schema_checker_->get_database_schema(ctx_->session_info_->get_effective_tenant_id(), + mv_schema->get_database_id(), + db_schema))) { + LOG_WARN("failed to get data base schema", K(ret), K(mv_schema->get_database_id())); + } else if (OB_ISNULL(db_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("database schema is NULL", K(ret), K(mv_schema->get_database_id())); + } else if (OB_FAIL(quick_rewrite_check(mv_schema, + query_rewrite_integrity == QueryRewriteIntegrityType::REWRITE_INTEGRITY_STALE_TOLERATED, + is_valid))) { + LOG_WARN("failed to do quick check", K(ret)); + } else if (!is_valid) { + // do nothing + } else { + uint64_t data_table_id = mv_schema->get_data_table_id(); + const ObTableSchema *data_table_schema = NULL; + if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), + data_table_id, + data_table_schema))) { + LOG_WARN("failed to get data table schema", K(ret), K(data_table_id)); + } else if (OB_ISNULL(data_table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data table schema is null", K(ret), K(mv_list.at(i))); + } else if (OB_FAIL(mv_infos_.push_back(MvInfo(mv_list.at(i), + data_table_id, + mv_schema, + data_table_schema, + db_schema, + NULL)))) { + LOG_WARN("failed to push back mv info", K(ret)); + } + } + } + return ret; +} + +int ObTransformMVRewrite::quick_rewrite_check(const ObTableSchema *mv_schema, + bool allow_stale, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + if (OB_ISNULL(mv_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv schema is null", K(ret)); + } else if (!mv_schema->mv_enable_query_rewrite()) { + is_valid = false; + } else if (!allow_stale && !mv_schema->mv_on_query_computation()) { + is_valid = false; + } else { + is_valid = true; + } + return false; +} + +int ObTransformMVRewrite::check_mv_stmt_basic_validity(const MvInfo &mv_info, + bool &is_valid) +{ + int ret = OB_SUCCESS; + const ObSelectStmt *stmt = NULL; + is_valid = false; + if (OB_ISNULL(stmt = mv_info.view_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_info)); + } else if (!stmt->is_spj()) { + is_valid = false; + OPT_TRACE("mv can not be used for rewrite"); + } else { + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::generate_mv_stmt(MvInfo &mv_info) +{ + int ret = OB_SUCCESS; + void* qctx_ptr = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->stmt_factory_) + || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(mv_info.mv_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv_info is null", K(ret), K(ctx_), KPC(mv_info.mv_schema_)); + } else { + SMART_VARS_2((ObTransformerCtx, trans_ctx), (ObResolverParams, resolver_ctx)) { + ObParser parser(*ctx_->allocator_, ctx_->session_info_->get_sql_mode(), ctx_->session_info_->get_charsets4parser()); + ParseResult parse_result; + ParseNode *node = NULL; + ObString view_definition; + ObDMLStmt *view_stmt = NULL; + uint64_t dummy_value = 0; + ObIAllocator &alloc = *ctx_->allocator_; + resolver_ctx.allocator_ = ctx_->allocator_; + resolver_ctx.schema_checker_ = ctx_->schema_checker_; + resolver_ctx.session_info_ = ctx_->session_info_; + resolver_ctx.expr_factory_ = ctx_->expr_factory_; + resolver_ctx.stmt_factory_ = ctx_->stmt_factory_; + resolver_ctx.sql_proxy_ = GCTX.sql_proxy_; + resolver_ctx.query_ctx_ = &mv_temp_query_ctx_; + resolver_ctx.is_for_rt_mv_ = true; + trans_ctx = *ctx_; + trans_ctx.reset(); + ObSelectResolver select_resolver(resolver_ctx); + ObTransformPreProcess transform_pre_process(&trans_ctx); + transform_pre_process.set_transformer_type(PRE_PROCESS); + STOP_OPT_TRACE; + if (OB_ISNULL(resolver_ctx.query_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query ctx is null", K(ret)); + } else if (OB_FAIL(ObSQLUtils::generate_view_definition_for_resolve(*ctx_->allocator_, + ctx_->session_info_->get_local_collation_connection(), + mv_info.mv_schema_->get_view_schema(), + view_definition))) { + LOG_WARN("fail to generate view definition for resolve", K(ret)); + } else if (OB_FAIL(parser.parse(view_definition, parse_result))) { + LOG_WARN("parse view definition failed", K(view_definition), K(ret)); + } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || OB_UNLIKELY(T_SELECT != node->type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv select node", K(ret), K(node), K(node->type_)); + } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ + = static_cast(parse_result.question_mark_ctx_.count_))) { + } else if (OB_FAIL(select_resolver.resolve(*node))) { + LOG_WARN("resolve view definition failed", K(ret)); + } else if (OB_ISNULL(view_stmt = static_cast(select_resolver.get_basic_stmt()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv stmt", K(ret), K(view_stmt)); + } else if (OB_FAIL(resolver_ctx.query_ctx_->query_hint_.init_query_hint(resolver_ctx.allocator_, + resolver_ctx.session_info_, + view_stmt))) { + LOG_WARN("failed to init query hint.", K(ret)); + } else if (OB_FAIL(resolver_ctx.query_ctx_->query_hint_.check_and_set_params_from_hint(resolver_ctx, + *view_stmt))) { + LOG_WARN("failed to check and set params from hint", K(ret)); + } else if (OB_FAIL(transform_pre_process.transform(view_stmt, dummy_value))) { + LOG_WARN("failed to do transform pre process", K(ret), KPC(view_stmt)); + } else if (OB_ISNULL(mv_info.view_stmt_ = static_cast(view_stmt))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv stmt", K(ret), K(view_stmt)); + } else { + LOG_DEBUG("generate mv stmt", KPC(mv_info.view_stmt_)); + } + RESUME_OPT_TRACE; + OPT_TRACE(mv_info.mv_schema_->get_table_name(), ":", view_stmt); + } + } + return ret; +} + +int ObTransformMVRewrite::do_transform(ObDMLStmt *&stmt, + bool &trans_happened) +{ + int ret = OB_SUCCESS; + const int64_t max_mv_stmt_gen = 10; + trans_happened = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(stmt) || OB_ISNULL(stmt->get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(stmt)); + } else if (!stmt->is_select_stmt()) { + // do nothing + } else { + ObSelectStmt *ori_stmt = static_cast(stmt); + ObSelectStmt *new_stmt = NULL; + const ObCollationType cs_type = stmt->get_query_ctx()->get_query_hint().cs_type_; + const ObMVRewriteHint *myhint = static_cast(get_hint(stmt->get_stmt_hint())); + for (int64_t i = 0; OB_SUCC(ret) && i < mv_infos_.count(); ++i) { + bool is_happened = false; + bool is_match_hint = true; + bool is_mv_valid = false; + MvInfo &mv_info = mv_infos_.at(i); + if (NULL != myhint && OB_FAIL(myhint->check_mv_match_hint(cs_type, + mv_info.mv_schema_, + mv_info.db_schema_, + is_match_hint))) { + LOG_WARN("failed to check mv match hint", K(ret)); + } else if (!is_match_hint) { + // do nothing + } else if (NULL == mv_info.view_stmt_ && ++mv_stmt_gen_count_ > max_mv_stmt_gen) { + // do nothing + } else if (NULL == mv_info.view_stmt_ && OB_FAIL(generate_mv_stmt(mv_info))) { + LOG_WARN("failed to generate mv stmt", K(ret), K(mv_info)); + } else if (OB_FAIL(check_mv_stmt_basic_validity(mv_info, is_mv_valid))) { + LOG_WARN("failed to check mv basic validity", K(ret), K(mv_info)); + } else if (!is_mv_valid) { + // do nothing + } else if (OB_FAIL(try_transform_with_one_mv(ori_stmt, mv_info, new_stmt, is_happened))) { + LOG_WARN("failed to try one mv", K(ret)); + } else if (is_happened) { + ori_stmt = new_stmt; + trans_happened = true; + break; + } + } + if (OB_SUCC(ret) && trans_happened) { + stmt = ori_stmt; + } + } + return ret; +} + +int ObTransformMVRewrite::try_transform_with_one_mv(ObSelectStmt *origin_stmt, + MvInfo &mv_info, + ObSelectStmt *&new_stmt, + bool &transform_happened) +{ + int ret = OB_SUCCESS; + ObStmtMapInfo map_info; + QueryRelation relation; + ObTryTransHelper try_trans_helper; + bool is_valid = false; + transform_happened = false; + new_stmt = NULL; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.mv_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info.mv_schema_)); + } else if (OB_FAIL(try_trans_helper.fill_helper(origin_stmt->get_query_ctx()))) { + LOG_WARN("failed to fill try trans helper", K(ret)); + } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(origin_stmt, + mv_info.view_stmt_, + map_info, + relation, + false, + false, + false))) { + LOG_WARN("failed to check stmt containment", K(ret)); + } else if ((QueryRelation::QUERY_EQUAL != relation + && QueryRelation::QUERY_LEFT_SUBSET != relation) + || map_info.left_can_be_replaced_ == false) { + OPT_TRACE(mv_info.mv_schema_->get_table_name(), ": stmt is not matched"); + } else if (OB_FAIL(do_transform_use_one_mv(origin_stmt, + mv_info, + map_info, + new_stmt, + is_valid))) { + LOG_WARN("failed to do transform use one mv", K(ret)); + } else if (!is_valid) { + OPT_TRACE(mv_info.mv_schema_->get_table_name(), ": stmt is not matched"); + } else if (OB_FAIL(check_rewrite_expected(origin_stmt, + new_stmt, + mv_info, + is_valid))) { + LOG_WARN("failed to check rewrite expected", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(add_param_constraint(map_info))) { + LOG_WARN("failed to add param constraint", K(ret)); + } else if (OB_FAIL(add_transform_hint(*new_stmt, &mv_info))) { + LOG_WARN("failed to add transform hint", K(ret)); + } else { + transform_happened = true; + } + if (OB_SUCC(ret) && !transform_happened + && OB_FAIL(try_trans_helper.recover(origin_stmt->get_query_ctx()))) { + LOG_WARN("failed to recover params", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::do_transform_use_one_mv(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + ObStmtMapInfo &map_info, + ObSelectStmt *&new_stmt, + bool &is_valid_transform) +{ + int ret = OB_SUCCESS; + is_valid_transform = false; + new_stmt = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) + || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_) + || OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.mv_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(mv_info.mv_schema_)); + } else { + bool is_valid = true; + GenerateStmtHelper helper(*ctx_->expr_factory_); + if (OB_FAIL(ctx_->stmt_factory_->create_stmt(helper.new_stmt_))) { + LOG_WARN("failed to create stmt", K(ret)); + } else if (OB_ISNULL(helper.new_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stmt is null", K(ret)); + } else if (OB_FALSE_IT(helper.new_stmt_->set_query_ctx(origin_stmt->get_query_ctx()))) { + } else if (OB_FAIL(helper.new_stmt_->get_stmt_hint().assign(origin_stmt->get_stmt_hint()))) { + LOG_WARN("failed to assign stmt hint", K(ret)); + } else if (OB_FAIL(helper.new_stmt_->adjust_statement_id(ctx_->allocator_, + ctx_->src_qb_name_, + ctx_->src_hash_val_))) { + LOG_WARN("failed to adjust statement id", K(ret)); + } else if (OB_FALSE_IT(helper.map_info_ = &map_info)) { + } else if (OB_FAIL(create_mv_table_item(mv_info, helper))) { + LOG_WARN("failed to create mv table item", K(ret), K(mv_info)); + } else if (OB_ISNULL(helper.mv_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv table item is null", K(ret)); + } else if (OB_FAIL(create_mv_column_item(mv_info, helper))) { + LOG_WARN("failed to create mv column item", K(ret)); + } else if (OB_FAIL(fill_from_item(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add from item", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_condition_exprs(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add condition exprs", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_groupby_exprs(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add group by exprs", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_select_item(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add select item", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_having_exprs(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add having exprs", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_distinct(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add distinct", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_orderby_exprs(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add order by exprs", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(fill_limit(origin_stmt, mv_info, helper, is_valid))) { + LOG_WARN("failed to add limit exprs", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(expand_rt_mv_table(helper))) { + LOG_WARN("failed to expand real time mv", K(ret)); + } else if (OB_FAIL(helper.new_stmt_->formalize_stmt(ctx_->session_info_))) { + LOG_WARN("failed to formalize stmt info", K(ret)); + } else { + new_stmt = helper.new_stmt_; + is_valid_transform = true; + OPT_TRACE("generate rewrite stmt use", mv_info.mv_schema_->get_table_name(), ":", new_stmt); + } + } + return ret; +} + +int ObTransformMVRewrite::create_mv_table_item(const MvInfo &mv_info, + GenerateStmtHelper &helper) +{ + int ret = OB_SUCCESS; + TableItem *mv_item = NULL; + ObString qb_name; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->schema_checker_) || + OB_ISNULL(ctx_->session_info_) || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.new_stmt_->get_query_ctx()) || + OB_ISNULL(mv_info.mv_schema_) || OB_ISNULL(mv_info.data_table_schema_) || OB_ISNULL(mv_info.db_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(helper.new_stmt_), K(mv_info)); + } else if (OB_FAIL(helper.new_stmt_->get_qb_name(qb_name))) { + } else if (OB_UNLIKELY(NULL == (mv_item = helper.new_stmt_->create_table_item(*ctx_->allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create table item failed", K(ret)); + } else { + mv_item->type_ = TableItem::ALIAS_TABLE; + // mv_item->synonym_name_ = ; + // mv_item->synonym_db_name_ = ; + mv_item->database_name_ = mv_info.db_schema_->get_database_name_str(); + mv_item->ref_id_ = mv_info.mv_schema_->get_data_table_id(); + mv_item->mview_id_ = mv_info.mv_schema_->get_table_id(); + mv_item->table_type_ = mv_info.mv_schema_->get_table_type(); + mv_item->table_id_ = helper.new_stmt_->get_query_ctx()->available_tb_id_--; + mv_item->is_index_table_ = mv_info.mv_schema_->is_index_table(); + mv_item->table_name_ = mv_info.data_table_schema_->get_table_name_str(); + mv_item->alias_name_ = mv_info.mv_schema_->get_table_name_str(); + mv_item->qb_name_ = qb_name; + + ObSchemaObjVersion table_version; + table_version.object_id_ = mv_info.mv_schema_->get_table_id(); + table_version.object_type_ = mv_info.mv_schema_->is_view_table() ? DEPENDENCY_VIEW : DEPENDENCY_TABLE; + table_version.version_ = mv_info.mv_schema_->get_schema_version(); + table_version.is_db_explicit_ = true; + uint64_t dep_db_id = mv_info.mv_schema_->get_database_id(); + + ObSchemaObjVersion data_table_version; + data_table_version.object_id_ = mv_info.data_table_schema_->get_table_id(); + data_table_version.object_type_ = DEPENDENCY_TABLE; + data_table_version.version_ = mv_info.data_table_schema_->get_schema_version(); + data_table_version.is_db_explicit_ = true; + uint64_t data_dep_db_id = mv_info.data_table_schema_->get_database_id(); + + if (OB_FAIL(helper.new_stmt_->add_global_dependency_table(table_version))) { + LOG_WARN("add global dependency table failed", K(ret)); + } else if (OB_FAIL(helper.new_stmt_->add_ref_obj_version(mv_info.mv_schema_->get_table_id(), + dep_db_id, + ObObjectType::VIEW, + table_version, + *ctx_->allocator_))) { + LOG_WARN("failed to add ref obj version", K(ret)); + } else if (OB_FAIL(helper.new_stmt_->add_global_dependency_table(data_table_version))) { + LOG_WARN("add global dependency table failed", K(ret)); + } else if (OB_FAIL(helper.new_stmt_->add_ref_obj_version(mv_info.mv_schema_->get_table_id(), + data_dep_db_id, + ObObjectType::VIEW, + data_table_version, + *ctx_->allocator_))) { + LOG_WARN("failed to add ref obj version", K(ret)); + } else if (OB_FAIL(helper.new_stmt_->add_table_item(ctx_->session_info_, mv_item))) { + LOG_WARN("push back table item failed", K(ret), KPC(mv_item)); + } else if (OB_FAIL(helper.new_stmt_->rebuild_tables_hash())) { + LOG_WARN("failed to rebuid table hash", K(ret)); + } + } + if (OB_SUCC(ret)) { + helper.mv_item_ = mv_item; + } + return ret; +} + +int ObTransformMVRewrite::create_mv_column_item(const MvInfo &mv_info, + GenerateStmtHelper &helper) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) || OB_ISNULL(ctx_->sql_schema_guard_) + || OB_ISNULL(ctx_->sql_schema_guard_->get_schema_guard()) || OB_ISNULL(ctx_->session_info_) + || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.mv_item_) + || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(mv_info.data_table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(helper.new_stmt_), K(helper.mv_item_), K(mv_info)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.view_stmt_->get_select_item_size(); ++i) { + const ObColumnSchemaV2 *col_schema = NULL; + ObColumnRefRawExpr *col_expr = NULL; + bool is_uni = false; + bool is_mul = false; + ColumnItem column_item; + if (OB_ISNULL(col_schema = mv_info.data_table_schema_->get_column_schema(OB_APP_MIN_COLUMN_ID + i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is null", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_column_expr(*ctx_->expr_factory_, *col_schema, col_expr))) { + LOG_WARN("build column expr failed", K(ret)); + } else if (OB_FAIL(mv_info.data_table_schema_->is_unique_key_column(*ctx_->sql_schema_guard_->get_schema_guard(), + col_schema->get_column_id(), + is_uni))) { + LOG_WARN("fail to check is unique key column", K(ret), KPC(mv_info.data_table_schema_), K(col_schema->get_column_id())); + } else if (OB_FAIL(mv_info.data_table_schema_->is_multiple_key_column(*ctx_->sql_schema_guard_->get_schema_guard(), + col_schema->get_column_id(), + is_mul))) { + LOG_WARN("fail to check is multiple key column", K(ret), KPC(mv_info.data_table_schema_), K(col_schema->get_column_id())); + } else { + if (!ob_enable_lob_locator_v2()) { + // Notice: clob will not convert to ObLobType if locator v2 enabled + if (is_oracle_mode() && ObLongTextType == col_expr->get_data_type() + && ! is_virtual_table(col_schema->get_table_id())) { + col_expr->set_data_type(ObLobType); + } + } + if (helper.mv_item_->alias_name_.empty()) { + col_expr->set_synonym_db_name(helper.mv_item_->synonym_db_name_); + col_expr->set_synonym_name(helper.mv_item_->synonym_name_); + } + col_expr->set_column_attr(helper.mv_item_->get_table_name(), col_schema->get_column_name_str()); + col_expr->set_from_alias_table(!helper.mv_item_->alias_name_.empty()); + col_expr->set_database_name(helper.mv_item_->database_name_); + //column maybe from alias table, so must reset ref id by table id from table_item + col_expr->set_ref_id(helper.mv_item_->table_id_, col_schema->get_column_id()); + col_expr->set_unique_key_column(is_uni); + col_expr->set_mul_key_column(is_mul); + if (!helper.mv_item_->alias_name_.empty()) { + col_expr->set_table_alias_name(); + } + col_expr->set_lob_column(is_lob_storage(col_schema->get_data_type())); + if (ctx_->session_info_->get_ddl_info().is_ddl()) { + column_item.set_default_value(col_schema->get_orig_default_value()); + } else { + column_item.set_default_value(col_schema->get_cur_default_value()); + } + column_item.expr_ = col_expr; + column_item.table_id_ = col_expr->get_table_id(); + column_item.column_id_ = col_expr->get_column_id(); + column_item.column_name_ = col_expr->get_column_name(); + column_item.base_tid_ = helper.mv_item_->ref_id_; + column_item.base_cid_ = column_item.column_id_; + column_item.is_geo_ = col_schema->is_geometry(); + LOG_DEBUG("succ to fill column_item", K(column_item), KPC(col_schema)); + if (OB_FAIL(helper.new_stmt_->add_column_item(column_item))) { + LOG_WARN("add column item to stmt failed", K(ret)); + } else if (OB_FAIL(col_expr->pull_relation_id())) { + LOG_WARN("failed to pullup relation ids", K(ret)); + } else if (OB_FAIL(helper.col_copier_.add_replaced_expr(mv_info.view_stmt_->get_select_item(i).expr_, col_expr))) { + LOG_WARN("failed to add replaced expr", K(ret), KPC(mv_info.view_stmt_->get_select_item(i).expr_), KPC(col_expr)); + } + } + } + return ret; +} + +int ObTransformMVRewrite::fill_from_item(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + TableItem *mv_item = NULL; + is_valid = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.mv_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(helper.new_stmt_), K(helper.mv_item_)); + } else if (OB_FAIL(helper.new_stmt_->add_from_item(helper.mv_item_->table_id_, false))) { + LOG_WARN("failed to add from item", K(ret), KPC(helper.new_stmt_), KPC(helper.mv_item_)); + } else { + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::fill_select_item(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray ori_select_exprs; + ObSEArray mv_select_exprs; + ObSEArray new_select_exprs; + bool is_sub_valid = false; + is_valid = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(origin_stmt) + || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(helper.new_stmt_) + || OB_ISNULL(helper.map_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(helper.new_stmt_), K(helper.map_info_)); + } else if (OB_FAIL(origin_stmt->get_select_exprs(ori_select_exprs))) { + LOG_WARN("failed to get origin select exprs", K(ret)); + } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { + LOG_WARN("failed to get mv select exprs", K(ret)); + } else if (OB_FAIL(ObStmtComparer::compute_new_expr(ori_select_exprs, + origin_stmt, + mv_select_exprs, + mv_info.view_stmt_, + *helper.map_info_, + helper.compute_expr_copier_, + new_select_exprs, + is_sub_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_sub_valid) { + LOG_TRACE("select exprs can not be computed", K(new_select_exprs)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < new_select_exprs.count(); ++i) { + ObRawExpr *select_expr = NULL; + if (OB_FAIL(helper.col_copier_.copy_on_replace(new_select_exprs.at(i), select_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(new_select_exprs.at(i))); + } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, + select_expr, + helper.new_stmt_))) { + LOG_WARN("failed to create select item", K(ret)); + } else if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(helper.new_stmt_, select_expr))) { + LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(select_expr)); + } + } + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::fill_condition_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray origin_unmatched_conds; + ObSEArray mv_unmatched_conds; + ObSEArray mv_select_exprs; + ObSEArray cond_exprs; + ObSEArray new_cond_exprs; + bool is_sub_valid = false; + is_valid = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) + || OB_ISNULL(helper.map_info_) || OB_ISNULL(helper.new_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.map_info_)); + } else if (OB_FAIL(ObStmtComparer::compute_unmatched_item(helper.map_info_->cond_map_, + origin_stmt->get_condition_size(), + mv_info.view_stmt_->get_condition_size(), + origin_unmatched_conds, + mv_unmatched_conds))) { + LOG_WARN("failed to compute unmatched item", K(ret)); + } else if (OB_FAIL(cond_exprs.prepare_allocate(origin_unmatched_conds.count()))) { + LOG_WARN("failed to pre-allocate table map", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < origin_unmatched_conds.count(); ++i) { + cond_exprs.at(i) = origin_stmt->get_condition_expr(origin_unmatched_conds.at(i)); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { + LOG_WARN("failed to get mv select exprs", K(ret)); + } else if (OB_FAIL(ObStmtComparer::compute_new_expr(cond_exprs, + origin_stmt, + mv_select_exprs, + mv_info.view_stmt_, + *helper.map_info_, + helper.compute_expr_copier_, + new_cond_exprs, + is_sub_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_sub_valid) { + LOG_TRACE("condition exprs can not be computed", K(new_cond_exprs)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < new_cond_exprs.count(); ++i) { + ObRawExpr *cond_expr = NULL; + if (OB_FAIL(helper.col_copier_.copy_on_replace(new_cond_exprs.at(i), cond_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(new_cond_exprs.at(i))); + } else if (OB_FAIL(helper.new_stmt_->add_condition_expr(cond_expr))) { + LOG_WARN("failed to add condition expr", K(ret), KPC(cond_expr)); + } + } + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::fill_having_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray origin_unmatched_conds; + ObSEArray mv_unmatched_conds; + ObSEArray mv_select_exprs; + ObSEArray new_having_exprs; + bool is_sub_valid = false; + is_valid = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) + || OB_ISNULL(helper.map_info_) || OB_ISNULL(helper.new_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.map_info_)); + } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { + LOG_WARN("failed to get mv select exprs", K(ret)); + } else if (OB_FAIL(ObStmtComparer::compute_new_expr(origin_stmt->get_having_exprs(), + origin_stmt, + mv_select_exprs, + mv_info.view_stmt_, + *helper.map_info_, + helper.compute_expr_copier_, + new_having_exprs, + is_sub_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_sub_valid) { + LOG_TRACE("having exprs can not be computed", K(new_having_exprs)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < new_having_exprs.count(); ++i) { + ObRawExpr *having_expr = NULL; + if (OB_FAIL(helper.col_copier_.copy_on_replace(new_having_exprs.at(i), having_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(new_having_exprs.at(i))); + } else if (OB_FAIL(helper.new_stmt_->add_having_expr(having_expr))) { + LOG_WARN("failed to add having expr", K(ret), KPC(having_expr)); + } else if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(helper.new_stmt_, having_expr))) { + LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(having_expr)); + } + } + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::fill_groupby_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + // TODO only mv has no group by + int ret = OB_SUCCESS; + ObSEArray mv_select_exprs; + ObSEArray new_groupby_exprs; + ObSEArray new_rollup_exprs; + bool is_sub_valid = false; + is_valid = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(origin_stmt) + || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(helper.new_stmt_) + || OB_ISNULL(helper.map_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); + } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { + LOG_WARN("failed to get mv select exprs", K(ret)); + } else if (OB_FAIL(ObStmtComparer::compute_new_expr(origin_stmt->get_group_exprs(), + origin_stmt, + mv_select_exprs, + mv_info.view_stmt_, + *helper.map_info_, + helper.compute_expr_copier_, + new_groupby_exprs, + is_sub_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_sub_valid) { + LOG_TRACE("group by exprs can not be computed", K(new_groupby_exprs)); + } else if (OB_FAIL(ObStmtComparer::compute_new_expr(origin_stmt->get_rollup_exprs(), + origin_stmt, + mv_select_exprs, + mv_info.view_stmt_, + *helper.map_info_, + helper.compute_expr_copier_, + new_rollup_exprs, + is_sub_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_sub_valid) { + // do nothing + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < new_groupby_exprs.count(); ++i) { + ObRawExpr *groupby_expr = NULL; + if (OB_FAIL(helper.col_copier_.copy_on_replace(new_groupby_exprs.at(i), groupby_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(new_groupby_exprs.at(i))); + } else if (OB_FAIL(helper.new_stmt_->get_group_exprs().push_back(groupby_expr))) { + LOG_WARN("failed to push back group by expr", K(ret), KPC(groupby_expr)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < new_rollup_exprs.count(); ++i) { + ObRawExpr *rollup_expr = NULL; + if (OB_FAIL(helper.col_copier_.copy_on_replace(new_rollup_exprs.at(i), rollup_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(new_rollup_exprs.at(i))); + } else if (OB_FAIL(helper.new_stmt_->get_rollup_exprs().push_back(rollup_expr))) { + LOG_WARN("failed to push back roll up expr", K(ret), KPC(rollup_expr)); + } + } + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::fill_distinct(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) + || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.map_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); + } else if (!origin_stmt->is_distinct()) { // here we assume that mv will not contain DISTINCT + is_valid = true; + } else if (origin_stmt->is_distinct()) { + helper.new_stmt_->assign_distinct(); + is_valid = true; + } else { + is_valid = false; + } + return ret; +} + +int ObTransformMVRewrite::fill_orderby_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray mv_select_exprs; + ObSEArray ori_orderby_exprs; + ObSEArray new_orderby_exprs; + bool is_sub_valid = false; + is_valid = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) + || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.map_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); + } else if (OB_FAIL(origin_stmt->get_order_exprs(ori_orderby_exprs))) { + LOG_WARN("failed to get order by exprs", K(ret)); + } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { + LOG_WARN("failed to get mv select exprs", K(ret)); + } else if (OB_FAIL(ObStmtComparer::compute_new_expr(ori_orderby_exprs, + origin_stmt, + mv_select_exprs, + mv_info.view_stmt_, + *helper.map_info_, + helper.compute_expr_copier_, + new_orderby_exprs, + is_sub_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_sub_valid) { + LOG_TRACE("order by exprs can not be computed", K(new_orderby_exprs)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < new_orderby_exprs.count(); ++i) { + ObRawExpr *orderby_expr = NULL; + OrderItem order_item; + if (OB_FAIL(helper.col_copier_.copy_on_replace(new_orderby_exprs.at(i), orderby_expr))) { + LOG_WARN("failed to copy expr", K(ret), KPC(new_orderby_exprs.at(i))); + } else if (OB_FALSE_IT(order_item.expr_ = orderby_expr)) { + } else if (OB_FALSE_IT(order_item.order_type_ = origin_stmt->get_order_item(i).order_type_)) { + } else if (OB_FAIL(helper.new_stmt_->add_order_item(order_item))) { + LOG_WARN("fail to add order item", K(ret), K(order_item)); + } else if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(helper.new_stmt_, orderby_expr))) { + LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(orderby_expr)); + } + } + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::fill_limit(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid) +{ + // TODO we assumed that the materialized view has no LIMIT + int ret = OB_SUCCESS; + is_valid = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) + || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.map_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); + } else if (!origin_stmt->has_limit()) { + is_valid = true; + } else { + helper.new_stmt_->get_limit_expr() = origin_stmt->get_limit_expr(); + helper.new_stmt_->get_offset_expr() = origin_stmt->get_offset_expr(); + helper.new_stmt_->get_limit_percent_expr() = origin_stmt->get_limit_percent_expr(); + helper.new_stmt_->set_has_fetch(origin_stmt->has_fetch()); + helper.new_stmt_->set_fetch_with_ties(origin_stmt->is_fetch_with_ties()); + is_valid = true; + } + return ret; +} + +int ObTransformMVRewrite::expand_rt_mv_table(GenerateStmtHelper &helper) +{ + int ret = OB_SUCCESS; + ObString src_qb_name; + int64_t query_rewrite_integrity = QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(helper.mv_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(helper.mv_item_)); + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_integrity(query_rewrite_integrity))) { + LOG_WARN("failed to get query rewrite integrity", K(ret)); + } else if (QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED != query_rewrite_integrity) { + // do nothing + } else if (OB_FALSE_IT(src_qb_name = ctx_->src_qb_name_)) { + } else if (OB_FALSE_IT(helper.mv_item_->need_expand_rt_mv_ = true)) { + } else if (OB_FAIL(ObTransformUtils::expand_mview_table(ctx_, helper.new_stmt_, helper.mv_item_))) { + LOG_WARN("failed to expand mv table", K(ret)); + } else { + ctx_->src_qb_name_ = src_qb_name; + } + return ret; +} + +int ObTransformMVRewrite::check_rewrite_expected(const ObSelectStmt *origin_stmt, + const ObSelectStmt *new_stmt, + const MvInfo &mv_info, + bool &is_expected) +{ + int ret = OB_SUCCESS; + bool hint_rewrite = false; + bool hint_no_rewrite = false; + int64_t query_rewrite_enabled = QueryRewriteEnabledType::REWRITE_ENABLED_TRUE; + bool is_match_index = false; + is_expected = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) + || OB_ISNULL(origin_stmt) || OB_ISNULL(new_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(new_stmt)); + } else if (OB_FAIL(check_hint_valid(*origin_stmt, hint_rewrite, hint_no_rewrite))) { + LOG_WARN("failed to check mv rewrite hint", K(ret)); + } else if (hint_rewrite) { + is_expected = true; + OPT_TRACE("hint force mv rewrite, skip cost check"); + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_enabled(query_rewrite_enabled))) { + LOG_WARN("failed to get query rewrite enabled", K(ret)); + } else if (QueryRewriteEnabledType::REWRITE_ENABLED_FORCE == query_rewrite_enabled) { + is_expected = true; + OPT_TRACE("system variable force mv rewrite, skip cost check"); + } else if (OB_FAIL(check_condition_match_index(new_stmt, is_match_index))) { + LOG_WARN("failed to check condition match index", K(ret)); + } else if (!is_match_index) { + is_expected = false; + OPT_TRACE("condition does not match index, can not rewrite"); + } else { + is_expected = true; + } + // TODO for the first version, we do not use the cost check + return ret; +} + +int ObTransformMVRewrite::check_condition_match_index(const ObSelectStmt *new_stmt, + bool &is_match_index) +{ + int ret = OB_SUCCESS; + is_match_index = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(new_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(new_stmt)); + } else if (0 == new_stmt->get_condition_size()) { + is_match_index = true; + } + for (int64_t i = 0; OB_SUCC(ret) && !is_match_index && i < new_stmt->get_condition_size(); ++i) { + ObSEArray column_exprs; + if (OB_FAIL(ObRawExprUtils::extract_column_exprs(new_stmt->get_condition_expr(i), column_exprs))) { + LOG_WARN("failed to extract column exprs", K(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && !is_match_index && j < column_exprs.count(); ++j) { + ObRawExpr *e = column_exprs.at(j); + ObColumnRefRawExpr *col_expr = NULL; + if (OB_ISNULL(e) || OB_UNLIKELY(!e->is_column_ref_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null expr", K(ret)); + } else if (OB_FALSE_IT(col_expr = static_cast(e))) { + } else if (OB_FAIL(ObTransformUtils::check_column_match_index(new_stmt, + new_stmt, + ctx_->sql_schema_guard_, + col_expr, + is_match_index))) { + LOG_WARN("failed to check column expr is match index", K(ret)); + } + } + } + return ret; +} + +int ObTransformMVRewrite::add_param_constraint(const ObStmtMapInfo &map_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(append_array_no_dup(ctx_->expr_constraints_, map_info.expr_cons_map_))) { + LOG_WARN("failed to add param constraint", K(ret)); + } else if (OB_FAIL(append_array_no_dup(ctx_->plan_const_param_constraints_, map_info.const_param_map_))) { + LOG_WARN("failed to add param constraint", K(ret)); + } else if (OB_FAIL(append_array_no_dup(ctx_->equal_param_constraints_, map_info.equal_param_map_))) { + LOG_WARN("failed to add param constraint", K(ret)); + } + return ret; +} + + +} //namespace sql +} //namespace oceanbase \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_mv_rewrite.h b/src/sql/rewrite/ob_transform_mv_rewrite.h new file mode 100644 index 000000000..f4fe3a35d --- /dev/null +++ b/src/sql/rewrite/ob_transform_mv_rewrite.h @@ -0,0 +1,224 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_TRANSFORM_MV_REWRITE_H +#define _OB_TRANSFORM_MV_REWRITE_H + +#include "sql/rewrite/ob_transform_rule.h" +#include "sql/rewrite/ob_stmt_comparer.h" +#include "sql/resolver/dml/ob_select_stmt.h" +#include "objit/common/ob_item_type.h" + +namespace oceanbase +{ + +namespace common +{ +class ObIAllocator; +template +class ObIArray; +}//common + +namespace sql +{ + +enum QueryRewriteEnabledType { + REWRITE_ENABLED_FALSE = 0, + REWRITE_ENABLED_TRUE , + REWRITE_ENABLED_FORCE +}; + +enum QueryRewriteIntegrityType { + REWRITE_INTEGRITY_ENFORCED = 0, + REWRITE_INTEGRITY_STALE_TOLERATED +}; + +/** + * @brief ObTransformMVRewrite + * + * Rewrite query with materialized view(s) + * + * e.g. + * create materialized view mv as + * select * from t_info join t_item on t_info.item_id = t_item.item_id; + * + * select * from t_info join t_item on t_info.item_id = t_item.item_id where user_id = 'xxx'; + * rewrite ==> + * select * from mv where user_id = 'xxx'; + */ +class ObTransformMVRewrite : public ObTransformRule +{ +public: + ObTransformMVRewrite(ObTransformerCtx *ctx) + : ObTransformRule(ctx, TransMethod::PRE_ORDER, T_MV_REWRITE), + is_mv_info_generated_(false), + mv_stmt_gen_count_(0) {} + virtual ~ObTransformMVRewrite() {} + virtual int transform_one_stmt(common::ObIArray &parent_stmts, + ObDMLStmt *&stmt, + bool &trans_happened) override; + virtual int construct_transform_hint(ObDMLStmt &stmt, void *trans_params) override; + +protected: + +private: + struct MvInfo { + MvInfo() : mv_id_(common::OB_INVALID_ID), + data_table_id_(common::OB_INVALID_ID), + mv_schema_(NULL), + data_table_schema_(NULL), + db_schema_(NULL), + view_stmt_(NULL) {} + + MvInfo(uint64_t mv_id, + uint64_t data_table_id, + const ObTableSchema *mv_schema, + const ObTableSchema *data_table_schema, + const ObDatabaseSchema *db_schema, + ObSelectStmt *view_stmt) + : mv_id_(mv_id), + data_table_id_(data_table_id), + mv_schema_(mv_schema), + data_table_schema_(data_table_schema), + db_schema_(db_schema), + view_stmt_(view_stmt) {} + + TO_STRING_KV( + K_(mv_id), + K_(data_table_id), + K_(mv_schema), + K_(data_table_schema), + K_(db_schema), + K_(view_stmt) + ); + + uint64_t mv_id_; + uint64_t data_table_id_; + const ObTableSchema *mv_schema_; + const ObTableSchema *data_table_schema_; + const ObDatabaseSchema *db_schema_; + ObSelectStmt *view_stmt_; + }; + + struct GenerateStmtHelper { + GenerateStmtHelper(ObRawExprFactory &expr_factory) : new_stmt_(NULL), + mv_item_(NULL), + map_info_(NULL), + relation_(QUERY_UNCOMPARABLE), + compute_expr_copier_(expr_factory), + col_copier_(expr_factory) {} + ObSelectStmt *new_stmt_; + TableItem *mv_item_; + ObStmtMapInfo *map_info_; + QueryRelation relation_; + ObRawExprCopier compute_expr_copier_; + ObRawExprCopier col_copier_; + }; + +private: + virtual int need_transform(const common::ObIArray &parent_stmts, + const int64_t current_level, + const ObDMLStmt &stmt, + bool &need_trans) override; + int check_hint_valid(const ObDMLStmt &stmt, + bool &force_rewrite, + bool &force_no_rewrite); + int check_table_has_mv(const ObDMLStmt &stmt, + bool &has_mv); + int check_basic_validity(const ObDMLStmt &stmt, + bool &is_valid); + int prepare_mv_info(const ObDMLStmt *root_stmt, + const ObDMLStmt *stmt); + int get_mv_list(const ObDMLStmt *root_stmt, + ObIArray &mv_list); + int get_base_table_id_string(const ObDMLStmt *stmt, + ObSqlString &table_ids); + int get_all_base_table_id(const ObDMLStmt *stmt, + ObIArray &table_ids); + int generate_mv_info(const ObDMLStmt *stmt, + ObIArray &mv_list); + int generate_mv_stmt(MvInfo &mv_info); + int quick_rewrite_check(const ObTableSchema *mv_schema, + bool allow_stale, + bool &is_valid); + int check_mv_stmt_basic_validity(const MvInfo &mv_info, + bool &is_valid); + int try_transform_with_one_mv(ObSelectStmt *origin_stmt, + MvInfo &mv_info, + ObSelectStmt *&new_stmt, + bool &transform_happened); + int do_transform(ObDMLStmt *&stmt, + bool &trans_happened); + int do_transform_use_one_mv(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + ObStmtMapInfo &map_info, + ObSelectStmt *&new_stmt, + bool &is_valid_transform); + int create_mv_table_item(const MvInfo &mv_info, + GenerateStmtHelper &helper); + int create_mv_column_item(const MvInfo &mv_info, + GenerateStmtHelper &helper); + int fill_from_item(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_select_item(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_condition_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_having_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_groupby_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_distinct(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_orderby_exprs(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int fill_limit(ObSelectStmt *origin_stmt, + const MvInfo &mv_info, + GenerateStmtHelper &helper, + bool &is_valid); + int expand_rt_mv_table(GenerateStmtHelper &helper); + int check_rewrite_expected(const ObSelectStmt *origin_stmt, + const ObSelectStmt *new_stmt, + const MvInfo &mv_info, + bool &is_expected); + int check_condition_match_index(const ObSelectStmt *new_stmt, + bool &is_match_index); + int add_param_constraint(const ObStmtMapInfo &map_info); + +private: + ObSEArray mv_infos_; + bool is_mv_info_generated_; + int64_t mv_stmt_gen_count_; + ObQueryCtx mv_temp_query_ctx_; // used for generating mv stmt + +private: + DISALLOW_COPY_AND_ASSIGN(ObTransformMVRewrite); + +}; + +} //namespace sql +} //namespace oceanbase +#endif \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 34db56712..8f795790d 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -77,6 +77,15 @@ int ObTransformPreProcess::transform_one_stmt(common::ObIArray OPT_TRACE("adjust duplicated table name", is_happened); LOG_TRACE("succeed to adjust duplicated table name", K(is_happened), K(ret)); + if (OB_SUCC(ret)) { + if (OB_FAIL(expand_materialized_view(stmt, is_happened))) { + LOG_WARN("failed to expand materialized view", K(ret)); + } else { + trans_happened |= is_happened; + OPT_TRACE("expand materialized view:", is_happened); + LOG_TRACE("succeed to expand materialized view",K(is_happened), K(ret)); + } + } if (OB_SUCC(ret)) { if (OB_FAIL(flatten_conditions(stmt, is_happened))) { LOG_WARN("failed to flatten_condition", K(ret)); @@ -332,6 +341,49 @@ int ObTransformPreProcess::need_transform(const common::ObIArraysession_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null", K(ret), K(stmt), K(ctx_)); + } else if (ctx_->session_info_->is_inner()) { + // when refresh mview, do not expand rt-mv + } else if (NULL != (hint = stmt->get_stmt_hint().get_normal_hint(T_MV_REWRITE)) + && hint->is_disable_hint()) { + // use no_mv_rewrite to disable expand rt mview + } else { + ObIArray &tables = stmt->get_table_items(); + TableItem *table_item = NULL; + bool is_modified = false; + for (int64_t i = 0; OB_SUCC(ret) && i < tables.count(); ++i) { + if (OB_ISNULL(table_item = tables.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null", K(ret), K(table_item)); + } else if (MATERIALIZED_VIEW != table_item->table_type_ + || !table_item->need_expand_rt_mv_) { + /* do nothing */ + } else if (OB_FAIL(stmt->check_table_be_modified(table_item->ref_id_, is_modified))) { + LOG_WARN("fail to check table be modified", K(ret)); + } else if (is_modified) { + /* do nothing */ + } else if (OB_UNLIKELY(!table_item->part_ids_.empty())) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "access some partitions of real-time materialized view now"); + LOG_WARN("access some partitions of real-time materialized view is not supported now", K(ret)); + } else if (OB_FAIL(ObTransformUtils::expand_mview_table(ctx_, stmt, table_item))) { + LOG_WARN("fail to expand mview table", K(ret)); + } else { + trans_happened = true; + } + } + } + return ret; +} + int ObTransformPreProcess::add_all_rowkey_columns_to_stmt(ObDMLStmt *stmt, bool &trans_happened) { int ret = OB_SUCCESS; diff --git a/src/sql/rewrite/ob_transform_pre_process.h b/src/sql/rewrite/ob_transform_pre_process.h index 0d160e832..bd1bdb2ea 100644 --- a/src/sql/rewrite/ob_transform_pre_process.h +++ b/src/sql/rewrite/ob_transform_pre_process.h @@ -638,6 +638,7 @@ struct DistinctObjMeta int flatten_conditions(ObDMLStmt *stmt, bool &trans_happened); int recursive_flatten_join_conditions(ObDMLStmt *stmt, TableItem *table, bool &trans_happened); int do_flatten_conditions(ObDMLStmt *stmt, ObIArray &conditions, bool &trans_happened); + int expand_materialized_view(ObDMLStmt *stmt, bool &trans_happened); int preserve_order_for_pagination(ObDMLStmt *stmt, bool &trans_happened); int check_stmt_need_preserve_order(ObDMLStmt *stmt, diff --git a/src/sql/rewrite/ob_transform_rule.cpp b/src/sql/rewrite/ob_transform_rule.cpp index 8a81fdb22..8e4fa5496 100644 --- a/src/sql/rewrite/ob_transform_rule.cpp +++ b/src/sql/rewrite/ob_transform_rule.cpp @@ -67,6 +67,7 @@ void ObTransformerCtx::reset() groupby_pushdown_stmts_.reset(); is_spm_outline_ = false; push_down_filters_.reset(); + iteration_level_ = 0; } int ObTransformerCtx::add_src_hash_val(const ObString &src_str) @@ -86,7 +87,7 @@ int ObTransformerCtx::add_src_hash_val(uint64_t trans_type) const char *str = NULL; if (OB_ISNULL(str = get_trans_type_string(trans_type))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to convert trans type to src value", K(ret)); + LOG_WARN("failed to convert trans type to src value", K(ret), K(trans_type)); } else { uint32_t hash_val = src_hash_val_.empty() ? 0 : src_hash_val_.at(src_hash_val_.count() - 1); hash_val = fnv_hash2(str, strlen(str), hash_val); @@ -135,6 +136,7 @@ const char* ObTransformerCtx::get_trans_type_string(uint64_t trans_type) TRANS_TYPE_TO_STR(SELECT_EXPR_PULLUP) TRANS_TYPE_TO_STR(PROCESS_DBLINK) TRANS_TYPE_TO_STR(DECORRELATE) + TRANS_TYPE_TO_STR(MV_REWRITE) default: return NULL; } } diff --git a/src/sql/rewrite/ob_transform_rule.h b/src/sql/rewrite/ob_transform_rule.h index e889de5d8..8648a25ea 100644 --- a/src/sql/rewrite/ob_transform_rule.h +++ b/src/sql/rewrite/ob_transform_rule.h @@ -16,6 +16,7 @@ #include "sql/resolver/dml/ob_raw_expr_sets.h" #include "sql/resolver/dml/ob_select_stmt.h" #include "sql/ob_optimizer_trace_impl.h" +#include "sql/ob_sql_context.h" namespace oceanbase { namespace common @@ -68,7 +69,8 @@ struct ObTransformerCtx used_trans_hints_(), groupby_pushdown_stmts_(), is_spm_outline_(false), - push_down_filters_() + push_down_filters_(), + iteration_level_(0) { } virtual ~ObTransformerCtx() {} @@ -129,6 +131,7 @@ struct ObTransformerCtx /* end used for hint and outline below */ bool is_spm_outline_; ObSEArray push_down_filters_; + uint64_t iteration_level_; }; enum TransMethod @@ -175,6 +178,7 @@ enum TRANSFORM_TYPE { SELECT_EXPR_PULLUP , PROCESS_DBLINK , DECORRELATE , + MV_REWRITE , TRANSFORM_TYPE_COUNT_PLUS_ONE , }; @@ -272,7 +276,8 @@ public: (1L << GROUPBY_PUSHDOWN) | (1L << GROUPBY_PULLUP) | (1L << SUBQUERY_COALESCE) | - (1L << SEMI_TO_INNER); + (1L << SEMI_TO_INNER) | + (1L << MV_REWRITE); ObTransformRule(ObTransformerCtx *ctx, TransMethod transform_method, diff --git a/src/sql/rewrite/ob_transform_subquery_coalesce.cpp b/src/sql/rewrite/ob_transform_subquery_coalesce.cpp index 88fa41ea8..423e3e154 100644 --- a/src/sql/rewrite/ob_transform_subquery_coalesce.cpp +++ b/src/sql/rewrite/ob_transform_subquery_coalesce.cpp @@ -502,82 +502,83 @@ int ObTransformSubqueryCoalesce::transform_diff_exprs( bool &rule_based_trans_happened) { int ret = OB_SUCCESS; - ObSEArray where_params; - ObSEArray having_params; bool where_is_false = false; bool having_is_false = false; bool hint_force_trans = false; ObSelectStmt *select_stmt = NULL; ObSelectStmt *select_trans_stmt = NULL; rule_based_trans_happened = false; - if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("params have null", K(ret), K(stmt), K(ctx_)); - } else if (stmt->is_select_stmt() && OB_ISNULL(select_stmt = static_cast(stmt))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("params have null", K(ret), K(stmt), K(select_stmt)); - } else if (OB_FAIL(check_conditions_validity(stmt, - stmt->get_condition_exprs(), - where_params, - where_is_false, - hint_force_trans))) { - LOG_WARN("failed to check condition validity", K(ret)); - } else if (OB_NOT_NULL(select_stmt) && - OB_FAIL(check_conditions_validity(stmt, - select_stmt->get_having_exprs(), - having_params, - having_is_false, - hint_force_trans))) { - LOG_WARN("failed to check having validity", K(ret)); - } else if (where_is_false && OB_FAIL(make_false(stmt->get_condition_exprs()))) { - LOG_WARN("failed to make condition false", K(ret)); - } else if (having_is_false && OB_FAIL(make_false(select_stmt->get_having_exprs()))) { - LOG_WARN("failed to make condition false", K(ret)); - } else if ((where_is_false || where_params.empty()) && - (having_is_false || having_params.empty())) { - // do nothing - } else if (!hint_force_trans && - OB_FAIL(ObTransformUtils::copy_stmt(*ctx_->stmt_factory_, stmt, trans_stmt))) { - LOG_WARN("failed to copy stmt", K(ret)); - } else if (hint_force_trans && - OB_FALSE_IT(trans_stmt = stmt)) { - } else if (trans_stmt->is_select_stmt() && - OB_ISNULL(select_trans_stmt = static_cast(trans_stmt))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("params have null", K(ret), K(trans_stmt), K(select_trans_stmt)); - } else if (!where_is_false && OB_FAIL(coalesce_diff_exists_exprs(trans_stmt, trans_stmt->get_condition_exprs(), where_params))) { - LOG_WARN("failed to do coalesce diff where conditions", K(ret)); - //bug: - // } else if (OB_NOT_NULL(select_trans_stmt) && !having_is_false && - // OB_FAIL(coalesce_diff_exists_exprs(trans_stmt, - // select_trans_stmt->get_having_exprs(), - // having_params))) { - // LOG_WARN("failed to do coalesce diff having conditions", K(ret)); - } else if (!where_is_false && OB_FAIL(coalesce_diff_any_all_exprs(trans_stmt, trans_stmt->get_condition_exprs(), where_params))) { - LOG_WARN("failed to do coalesce diff where conditions", K(ret)); - } else if (OB_NOT_NULL(select_trans_stmt) && !having_is_false && - OB_FAIL(coalesce_diff_any_all_exprs(trans_stmt, - select_trans_stmt->get_having_exprs(), - having_params))) { - LOG_WARN("failed to do coalesce diff having conditions", K(ret)); - } - if (OB_SUCC(ret)) { - rule_based_trans_happened = where_is_false || having_is_false || hint_force_trans; - ObIArray *where_equal_infos = where_is_false ? - &rule_based_equal_infos : &cost_based_equal_infos; - ObIArray *having_equal_infos = having_is_false ? - &rule_based_equal_infos : &cost_based_equal_infos; - for (int64_t i = 0; OB_SUCC(ret) && i < where_params.count(); ++i) { - if (OB_FAIL(append(*where_equal_infos, where_params.at(i).map_info_.equal_param_map_))) { - LOG_WARN("failed to append equal infos", K(ret)); + typedef ObSEArray ParamArray; + SMART_VARS_2((ParamArray, where_params), (ParamArray, having_params)) { + if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params have null", K(ret), K(stmt), K(ctx_)); + } else if (stmt->is_select_stmt() && OB_ISNULL(select_stmt = static_cast(stmt))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params have null", K(ret), K(stmt), K(select_stmt)); + } else if (OB_FAIL(check_conditions_validity(stmt, + stmt->get_condition_exprs(), + where_params, + where_is_false, + hint_force_trans))) { + LOG_WARN("failed to check condition validity", K(ret)); + } else if (OB_NOT_NULL(select_stmt) && + OB_FAIL(check_conditions_validity(stmt, + select_stmt->get_having_exprs(), + having_params, + having_is_false, + hint_force_trans))) { + LOG_WARN("failed to check having validity", K(ret)); + } else if (where_is_false && OB_FAIL(make_false(stmt->get_condition_exprs()))) { + LOG_WARN("failed to make condition false", K(ret)); + } else if (having_is_false && OB_FAIL(make_false(select_stmt->get_having_exprs()))) { + LOG_WARN("failed to make condition false", K(ret)); + } else if ((where_is_false || where_params.empty()) && + (having_is_false || having_params.empty())) { + // do nothing + } else if (!hint_force_trans && + OB_FAIL(ObTransformUtils::copy_stmt(*ctx_->stmt_factory_, stmt, trans_stmt))) { + LOG_WARN("failed to copy stmt", K(ret)); + } else if (hint_force_trans && + OB_FALSE_IT(trans_stmt = stmt)) { + } else if (trans_stmt->is_select_stmt() && + OB_ISNULL(select_trans_stmt = static_cast(trans_stmt))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params have null", K(ret), K(trans_stmt), K(select_trans_stmt)); + } else if (!where_is_false && OB_FAIL(coalesce_diff_exists_exprs(trans_stmt, trans_stmt->get_condition_exprs(), where_params))) { + LOG_WARN("failed to do coalesce diff where conditions", K(ret)); + //bug: + // } else if (OB_NOT_NULL(select_trans_stmt) && !having_is_false && + // OB_FAIL(coalesce_diff_exists_exprs(trans_stmt, + // select_trans_stmt->get_having_exprs(), + // having_params))) { + // LOG_WARN("failed to do coalesce diff having conditions", K(ret)); + } else if (!where_is_false && OB_FAIL(coalesce_diff_any_all_exprs(trans_stmt, trans_stmt->get_condition_exprs(), where_params))) { + LOG_WARN("failed to do coalesce diff where conditions", K(ret)); + } else if (OB_NOT_NULL(select_trans_stmt) && !having_is_false && + OB_FAIL(coalesce_diff_any_all_exprs(trans_stmt, + select_trans_stmt->get_having_exprs(), + having_params))) { + LOG_WARN("failed to do coalesce diff having conditions", K(ret)); + } + if (OB_SUCC(ret)) { + rule_based_trans_happened = where_is_false || having_is_false || hint_force_trans; + ObIArray *where_equal_infos = where_is_false ? + &rule_based_equal_infos : &cost_based_equal_infos; + ObIArray *having_equal_infos = having_is_false ? + &rule_based_equal_infos : &cost_based_equal_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < where_params.count(); ++i) { + if (OB_FAIL(append(*where_equal_infos, where_params.at(i).map_info_.equal_param_map_))) { + LOG_WARN("failed to append equal infos", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < having_params.count(); ++i) { + if (OB_FAIL(append(*having_equal_infos, having_params.at(i).map_info_.equal_param_map_))) { + LOG_WARN("failed to append equal infos", K(ret)); + } } } - for (int64_t i = 0; OB_SUCC(ret) && i < having_params.count(); ++i) { - if (OB_FAIL(append(*having_equal_infos, having_params.at(i).map_info_.equal_param_map_))) { - LOG_WARN("failed to append equal infos", K(ret)); - } - } - } + } // end smart vars return ret; } @@ -1935,138 +1936,140 @@ int ObTransformSubqueryCoalesce::inner_coalesce_subquery(ObSelectStmt *subquery, // if the select items have same udf, need check if they are deterministic // eg: select func(c1), func(c1) from t1; bool need_check_deterministic = true; - ObStmtCompareContext context(coalesce_query, - subquery, - map_info, - &query_ctx->calculable_items_, - need_check_deterministic); - if (OB_ISNULL(subquery) || OB_ISNULL(coalesce_query) || - OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || - OB_ISNULL(query_ctx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null param", K(ret)); - } else if (OB_FAIL(subquery->get_select_exprs(subquery_select_list))) { - LOG_WARN("failed to get select exprs", K(ret)); - } else if (OB_FAIL(subquery->get_column_exprs(subquery_column_list))) { - LOG_WARN("failed to get column exprs", K(ret)); - } else if (OB_FAIL(coalesce_query->get_select_exprs(coalesce_select_list))) { - LOG_WARN("failed to get select exprs", K(ret)); - } else if (OB_FAIL(coalesce_query->get_column_exprs(coalesce_column_list))) { - LOG_WARN("failed to get column exprs", K(ret)); - } - //check column item - for (int64_t i = 0; OB_SUCC(ret) && i < subquery_column_list.count(); ++i) { - ObRawExpr *subquery_column = subquery_column_list.at(i); - bool find = false; - if (OB_ISNULL(subquery_column)) { + SMART_VAR(ObStmtCompareContext, context, + coalesce_query, + subquery, + map_info, + &query_ctx->calculable_items_, + need_check_deterministic) { + if (OB_ISNULL(subquery) || OB_ISNULL(coalesce_query) || + OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || + OB_ISNULL(query_ctx)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null column expr", K(ret)); + LOG_WARN("unexpect null param", K(ret)); + } else if (OB_FAIL(subquery->get_select_exprs(subquery_select_list))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else if (OB_FAIL(subquery->get_column_exprs(subquery_column_list))) { + LOG_WARN("failed to get column exprs", K(ret)); + } else if (OB_FAIL(coalesce_query->get_select_exprs(coalesce_select_list))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else if (OB_FAIL(coalesce_query->get_column_exprs(coalesce_column_list))) { + LOG_WARN("failed to get column exprs", K(ret)); } - //Check if the column is already in the combined subquery - for (int64_t j = 0; OB_SUCC(ret) && !find && j < coalesce_column_list.count(); ++j) { - ObRawExpr *coalesce_column = coalesce_column_list.at(j); - if (OB_ISNULL(coalesce_column)) { + //check column item + for (int64_t i = 0; OB_SUCC(ret) && i < subquery_column_list.count(); ++i) { + ObRawExpr *subquery_column = subquery_column_list.at(i); + bool find = false; + if (OB_ISNULL(subquery_column)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null column expr", K(ret)); - } else if (!coalesce_column->same_as(*subquery_column, &context)) { - //do nothing - } else if (OB_FAIL(new_column_list.push_back(coalesce_column))) { - LOG_WARN("failed to push back expr", K(ret)); - } else { - find = true; + } + //Check if the column is already in the combined subquery + for (int64_t j = 0; OB_SUCC(ret) && !find && j < coalesce_column_list.count(); ++j) { + ObRawExpr *coalesce_column = coalesce_column_list.at(j); + if (OB_ISNULL(coalesce_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null column expr", K(ret)); + } else if (!coalesce_column->same_as(*subquery_column, &context)) { + //do nothing + } else if (OB_FAIL(new_column_list.push_back(coalesce_column))) { + LOG_WARN("failed to push back expr", K(ret)); + } else { + find = true; + } + } + //If not, it needs to be added to the combined subquery + if (OB_SUCC(ret) && !find) { + ColumnItem *column_item = NULL; + ObColumnRefRawExpr *col_ref = static_cast(subquery_column); + uint64_t table_id = OB_INVALID_ID; + if (!subquery_column->is_column_ref_expr()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expect column ref expr", KPC(subquery_column), K(ret)); + } else if (OB_ISNULL(column_item = subquery->get_column_item_by_id(col_ref->get_table_id(), + col_ref->get_column_id()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null column item", K(ret)); + } else if (OB_FAIL(get_map_table_id(subquery, + coalesce_query, + map_info, + col_ref->get_table_id(), + table_id))) { + LOG_WARN("failed to get map table id", K(ret)); + } else if (OB_FALSE_IT(column_item->table_id_ = table_id)) { + } else if (OB_FALSE_IT(col_ref->set_table_id(table_id))) { + } else if (OB_FAIL(new_column_items.push_back(*column_item))) { + LOG_WARN("failed to push back column item", K(ret)); + } else if (OB_FAIL(new_column_list.push_back(subquery_column))) { + LOG_WARN("failed to push back expr", K(ret)); + } } } - //If not, it needs to be added to the combined subquery - if (OB_SUCC(ret) && !find) { - ColumnItem *column_item = NULL; - ObColumnRefRawExpr *col_ref = static_cast(subquery_column); - uint64_t table_id = OB_INVALID_ID; - if (!subquery_column->is_column_ref_expr()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expect column ref expr", KPC(subquery_column), K(ret)); - } else if (OB_ISNULL(column_item = subquery->get_column_item_by_id(col_ref->get_table_id(), - col_ref->get_column_id()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null column item", K(ret)); - } else if (OB_FAIL(get_map_table_id(subquery, - coalesce_query, - map_info, - col_ref->get_table_id(), - table_id))) { - LOG_WARN("failed to get map table id", K(ret)); - } else if (OB_FALSE_IT(column_item->table_id_ = table_id)) { - } else if (OB_FALSE_IT(col_ref->set_table_id(table_id))) { - } else if (OB_FAIL(new_column_items.push_back(*column_item))) { - LOG_WARN("failed to push back column item", K(ret)); - } else if (OB_FAIL(new_column_list.push_back(subquery_column))) { - LOG_WARN("failed to push back expr", K(ret)); + if (OB_SUCC(ret) && !new_column_items.empty()) { + if (OB_FAIL(coalesce_query->add_column_item(new_column_items))) { + LOG_WARN("failed to add table item", K(ret)); } } - } - if (OB_SUCC(ret) && !new_column_items.empty()) { - if (OB_FAIL(coalesce_query->add_column_item(new_column_items))) { - LOG_WARN("failed to add table item", K(ret)); - } - } - //check select item - for (int64_t i = 0; OB_SUCC(ret) && i < subquery_select_list.count(); ++i) { - ObRawExpr *subquery_select = subquery_select_list.at(i); - bool find = false; - if (OB_ISNULL(subquery_select)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null select expr", K(ret)); - } else if (OB_FAIL(select_exprs.push_back(subquery_select))) { - LOG_WARN("failed to push back expr", K(ret)); - } - for (int64_t j = 0; OB_SUCC(ret) && !find && j < coalesce_select_list.count(); ++j) { - ObRawExpr *coalesce_select = coalesce_select_list.at(j); - if (OB_ISNULL(coalesce_select)) { + //check select item + for (int64_t i = 0; OB_SUCC(ret) && i < subquery_select_list.count(); ++i) { + ObRawExpr *subquery_select = subquery_select_list.at(i); + bool find = false; + if (OB_ISNULL(subquery_select)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null select expr", K(ret)); - } else if (!coalesce_select->same_as(*subquery_select, &context)) { - // do nothing - } else if (!is_first_subquery && - OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, - coalesce_select, - coalesce_query))) { - LOG_WARN("failed to create column for subquery", K(ret)); - } else if (OB_FAIL(index_map.push_back(is_first_subquery ? j : coalesce_query->get_select_item_size() - 1))) { - LOG_WARN("failed to push back index", K(ret)); - } else { - find = true; + } else if (OB_FAIL(select_exprs.push_back(subquery_select))) { + LOG_WARN("failed to push back expr", K(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && !find && j < coalesce_select_list.count(); ++j) { + ObRawExpr *coalesce_select = coalesce_select_list.at(j); + if (OB_ISNULL(coalesce_select)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null select expr", K(ret)); + } else if (!coalesce_select->same_as(*subquery_select, &context)) { + // do nothing + } else if (!is_first_subquery && + OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, + coalesce_select, + coalesce_query))) { + LOG_WARN("failed to create column for subquery", K(ret)); + } else if (OB_FAIL(index_map.push_back(is_first_subquery ? j : coalesce_query->get_select_item_size() - 1))) { + LOG_WARN("failed to push back index", K(ret)); + } else { + find = true; + } + } + if (OB_SUCC(ret) && !find) { + ObSEArray aggr_items; + ObSEArray win_func_exprs; + if (OB_FAIL(ObTransformUtils::replace_expr(subquery_column_list, new_column_list, subquery_select))) { + LOG_WARN("failed to replace expr", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(subquery_select, + aggr_items))) { + LOG_WARN("failed to extract aggr expr", K(ret)); + } else if (OB_FAIL(append(coalesce_query->get_aggr_items(), aggr_items))) { + LOG_WARN("failed to append aggr items", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_winfun_expr(subquery_select, win_func_exprs))) { + LOG_WARN("failed to extract win func exprs", K(ret)); + } else if (OB_FAIL(append(coalesce_query->get_window_func_exprs(), win_func_exprs))) { + LOG_WARN("failed to append win func exprs", K(ret)); + } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, + subquery_select, + coalesce_query))) { + LOG_WARN("failed to create column for subquery", K(ret)); + } else if (OB_FAIL(index_map.push_back(coalesce_query->get_select_item_size() - 1))) { + LOG_WARN("failed to push back index", K(ret)); + } } } - if (OB_SUCC(ret) && !find) { - ObSEArray aggr_items; - ObSEArray win_func_exprs; - if (OB_FAIL(ObTransformUtils::replace_expr(subquery_column_list, new_column_list, subquery_select))) { - LOG_WARN("failed to replace expr", K(ret)); - } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(subquery_select, - aggr_items))) { - LOG_WARN("failed to extract aggr expr", K(ret)); - } else if (OB_FAIL(append(coalesce_query->get_aggr_items(), aggr_items))) { - LOG_WARN("failed to append aggr items", K(ret)); - } else if (OB_FAIL(ObTransformUtils::extract_winfun_expr(subquery_select, win_func_exprs))) { - LOG_WARN("failed to extract win func exprs", K(ret)); - } else if (OB_FAIL(append(coalesce_query->get_window_func_exprs(), win_func_exprs))) { - LOG_WARN("failed to append win func exprs", K(ret)); - } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, - subquery_select, - coalesce_query))) { - LOG_WARN("failed to create column for subquery", K(ret)); - } else if (OB_FAIL(index_map.push_back(coalesce_query->get_select_item_size() - 1))) { - LOG_WARN("failed to push back index", K(ret)); + if (OB_SUCC(ret)) { + if (OB_FAIL(coalesce_query->adjust_subquery_list())) { + LOG_WARN("failed to adjust subquery list", K(ret)); + } else if (OB_FAIL(append(query_ctx->all_equal_param_constraints_, + context.equal_param_info_))) { + LOG_WARN("failed to append equal param constraints", K(ret)); } } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(coalesce_query->adjust_subquery_list())) { - LOG_WARN("failed to adjust subquery list", K(ret)); - } else if (OB_FAIL(append(query_ctx->all_equal_param_constraints_, - context.equal_param_info_))) { - LOG_WARN("failed to append equal param constraints", K(ret)); - } - } + } // end smart var return ret; } diff --git a/src/sql/rewrite/ob_transform_temp_table.cpp b/src/sql/rewrite/ob_transform_temp_table.cpp index 7f5538148..db97d5989 100644 --- a/src/sql/rewrite/ob_transform_temp_table.cpp +++ b/src/sql/rewrite/ob_transform_temp_table.cpp @@ -167,7 +167,8 @@ int ObTransformTempTable::generate_with_clause(ObDMLStmt *&stmt, bool &trans_hap OPT_TRACE("stmt has for update, can not extract CTE"); } else if (OB_FAIL(parent_map.create(128, "TempTable"))) { LOG_WARN("failed to init stmt map", K(ret)); - } else if (OB_FAIL(ObTransformUtils::get_all_child_stmts(stmt, child_stmts, &parent_map))) { + } else if (!ObOptimizerUtil::find_item(ctx_->temp_table_ignore_stmts_, stmt) && + OB_FAIL(ObTransformUtils::get_all_child_stmts(stmt, child_stmts, &parent_map, &ctx_->temp_table_ignore_stmts_))) { LOG_WARN("failed to get all child stmts", K(ret)); } else if (OB_FAIL(get_non_correlated_subquery(stmt, non_correlated_stmts))) { LOG_WARN("failed to get non correlated subquery", K(ret)); @@ -1025,8 +1026,6 @@ int ObTransformTempTable::remove_simple_stmts(ObIArray &stmts) LOG_WARN("failed to check has rownum", K(ret)); } else if (has_rownum) { //do nothing - } else if (ObOptimizerUtil::find_item(ctx_->temp_table_ignore_stmts_, subquery)) { - //do nothing } else if (OB_FAIL(check_stmt_can_materialize(subquery, false, is_valid))) { LOG_WARN("failed to check stmt is valid", K(ret)); } else if (!is_valid) { @@ -1590,7 +1589,6 @@ int ObTransformTempTable::apply_temp_table(ObSelectStmt *parent_stmt, ObStmtMapInfo& map_info) { int ret = OB_SUCCESS; - ObStmtCompareContext context; //视图的select items ObSEArray view_select_list; //视图的select items转换为temp table对应的select items @@ -1607,154 +1605,156 @@ int ObTransformTempTable::apply_temp_table(ObSelectStmt *parent_stmt, ObSEArray new_column_items; ObSEArray old_column_exprs; ObSelectStmt *view = NULL; - if (OB_ISNULL(parent_stmt) || OB_ISNULL(temp_table_query) || - OB_ISNULL(view_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null param", K(ret)); - } else if (!view_table->is_generated_table()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expect generate table", KPC(view_table), K(ret)); - } else if (OB_ISNULL(view = view_table->ref_query_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null ref query", KPC(view_table), K(ret)); - } else if (OB_FAIL(view->get_select_exprs(view_select_list))) { - LOG_WARN("failed to get select exprs", K(ret)); - } else if (OB_FAIL(view->get_column_exprs(view_column_list))) { - LOG_WARN("failed to get column exprs", K(ret)); - } else if (OB_FAIL(temp_table_query->get_select_exprs(temp_table_select_list))) { - LOG_WARN("failed to get select exprs", K(ret)); - } else if (OB_FAIL(temp_table_query->get_column_exprs(temp_table_column_list))) { - LOG_WARN("failed to get column exprs", K(ret)); - } else { - context.init(temp_table_query, view, map_info, - &parent_stmt->get_query_ctx()->calculable_items_); - } - //找到对应的column item，不存在于temp table的column需要添加到temp table - for (int64_t i = 0; OB_SUCC(ret) && i < view_column_list.count(); ++i) { - ObRawExpr *view_column = view_column_list.at(i); - bool find = false; - if (OB_ISNULL(view_column)) { + SMART_VAR(ObStmtCompareContext, context) { + if (OB_ISNULL(parent_stmt) || OB_ISNULL(temp_table_query) || + OB_ISNULL(view_table)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null column expr", K(ret)); + LOG_WARN("unexpect null param", K(ret)); + } else if (!view_table->is_generated_table()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expect generate table", KPC(view_table), K(ret)); + } else if (OB_ISNULL(view = view_table->ref_query_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null ref query", KPC(view_table), K(ret)); + } else if (OB_FAIL(view->get_select_exprs(view_select_list))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else if (OB_FAIL(view->get_column_exprs(view_column_list))) { + LOG_WARN("failed to get column exprs", K(ret)); + } else if (OB_FAIL(temp_table_query->get_select_exprs(temp_table_select_list))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else if (OB_FAIL(temp_table_query->get_column_exprs(temp_table_column_list))) { + LOG_WARN("failed to get column exprs", K(ret)); + } else { + context.init(temp_table_query, view, map_info, + &parent_stmt->get_query_ctx()->calculable_items_); } - //column item是否存在于temp table中 - for (int64_t j = 0; OB_SUCC(ret) && !find && j < temp_table_column_list.count(); ++j) { - ObRawExpr *temp_table_column = temp_table_column_list.at(j); - if (OB_ISNULL(temp_table_column)) { + //找到对应的column item，不存在于temp table的column需要添加到temp table + for (int64_t i = 0; OB_SUCC(ret) && i < view_column_list.count(); ++i) { + ObRawExpr *view_column = view_column_list.at(i); + bool find = false; + if (OB_ISNULL(view_column)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null column expr", K(ret)); - } else if (!temp_table_column->same_as(*view_column, &context)) { - //do nothing - } else if (OB_FAIL(new_column_list.push_back(temp_table_column))) { - LOG_WARN("failed to push back expr", K(ret)); - } else { - find = true; + } + //column item是否存在于temp table中 + for (int64_t j = 0; OB_SUCC(ret) && !find && j < temp_table_column_list.count(); ++j) { + ObRawExpr *temp_table_column = temp_table_column_list.at(j); + if (OB_ISNULL(temp_table_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null column expr", K(ret)); + } else if (!temp_table_column->same_as(*view_column, &context)) { + //do nothing + } else if (OB_FAIL(new_column_list.push_back(temp_table_column))) { + LOG_WARN("failed to push back expr", K(ret)); + } else { + find = true; + } + } + //不存在于temp table中的column需要添加到temp table中 + if (OB_SUCC(ret) && !find) { + TableItem *table = NULL; + ColumnItem *column_item = NULL; + ObColumnRefRawExpr *col_ref = static_cast(view_column); + uint64_t table_id = OB_INVALID_ID; + if (!view_column->is_column_ref_expr()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expect column ref expr", KPC(view_column), K(ret)); + } else if (OB_ISNULL(column_item = view->get_column_item_by_id(col_ref->get_table_id(), + col_ref->get_column_id()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null column item", K(ret)); + } else if (OB_FAIL(get_map_table_id(view, + temp_table_query, + map_info, + col_ref->get_table_id(), + table_id))) { + LOG_WARN("failed to get map table id", K(ret)); + } else if (OB_FALSE_IT(column_item->table_id_ = table_id)) { + } else if (OB_FALSE_IT(col_ref->set_table_id(table_id))) { + } else if (OB_FAIL(new_column_items.push_back(*column_item))) { + LOG_WARN("failed to push back column item", K(ret)); + } else if (OB_FAIL(new_column_list.push_back(view_column))) { + LOG_WARN("failed to push back expr", K(ret)); + } else if (OB_ISNULL(table = temp_table_query->get_table_item_by_id(table_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null table item", K(ret)); + } else { + col_ref->set_table_name(table->get_table_name()); + } } } - //不存在于temp table中的column需要添加到temp table中 - if (OB_SUCC(ret) && !find) { - TableItem *table = NULL; - ColumnItem *column_item = NULL; - ObColumnRefRawExpr *col_ref = static_cast(view_column); - uint64_t table_id = OB_INVALID_ID; - if (!view_column->is_column_ref_expr()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expect column ref expr", KPC(view_column), K(ret)); - } else if (OB_ISNULL(column_item = view->get_column_item_by_id(col_ref->get_table_id(), - col_ref->get_column_id()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null column item", K(ret)); - } else if (OB_FAIL(get_map_table_id(view, - temp_table_query, - map_info, - col_ref->get_table_id(), - table_id))) { - LOG_WARN("failed to get map table id", K(ret)); - } else if (OB_FALSE_IT(column_item->table_id_ = table_id)) { - } else if (OB_FALSE_IT(col_ref->set_table_id(table_id))) { - } else if (OB_FAIL(new_column_items.push_back(*column_item))) { - LOG_WARN("failed to push back column item", K(ret)); - } else if (OB_FAIL(new_column_list.push_back(view_column))) { - LOG_WARN("failed to push back expr", K(ret)); - } else if (OB_ISNULL(table = temp_table_query->get_table_item_by_id(table_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null table item", K(ret)); - } else { - col_ref->set_table_name(table->get_table_name()); + //添加新的column item + if (OB_SUCC(ret) && !new_column_items.empty()) { + if (OB_FAIL(temp_table_query->add_column_item(new_column_items))) { + LOG_WARN("failed to add table item", K(ret)); } } - } - //添加新的column item - if (OB_SUCC(ret) && !new_column_items.empty()) { - if (OB_FAIL(temp_table_query->add_column_item(new_column_items))) { - LOG_WARN("failed to add table item", K(ret)); - } - } - //找到不同的select item - for (int64_t i = 0; OB_SUCC(ret) && i < view_select_list.count(); ++i) { - ObRawExpr *view_select = view_select_list.at(i); - ObColumnRefRawExpr *col_expr = NULL; - bool find = false; - if (OB_ISNULL(view_select)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null select expr", K(ret)); - } else if (NULL == (col_expr = parent_stmt->get_column_expr_by_id(view_table->table_id_, - i + OB_APP_MIN_COLUMN_ID))) { - // unused select item, skip following procedure - find = true; - } else if (OB_FAIL(old_column_exprs.push_back(col_expr))) { - LOG_WARN("failed to push back expr", K(ret)); - } - //select item是否存在于temp table中 - for (int64_t j = 0; OB_SUCC(ret) && !find && j < temp_table_select_list.count(); ++j) { - ObRawExpr *temp_table_select = temp_table_select_list.at(j); - if (OB_ISNULL(temp_table_select)) { + //找到不同的select item + for (int64_t i = 0; OB_SUCC(ret) && i < view_select_list.count(); ++i) { + ObRawExpr *view_select = view_select_list.at(i); + ObColumnRefRawExpr *col_expr = NULL; + bool find = false; + if (OB_ISNULL(view_select)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null select expr", K(ret)); - } else if (!temp_table_select->same_as(*view_select, &context)) { - //do nothing - } else if (OB_FAIL(new_select_list.push_back(temp_table_select))) { - LOG_WARN("failed to push back expr", K(ret)); - } else { + } else if (NULL == (col_expr = parent_stmt->get_column_expr_by_id(view_table->table_id_, + i + OB_APP_MIN_COLUMN_ID))) { + // unused select item, skip following procedure find = true; - } - } - //不存在于temp table中的select expr需要转换成temp table的select item - if (OB_SUCC(ret) && !find) { - ObSEArray aggr_items; - ObSEArray win_func_exprs; - if (OB_FAIL(ObTransformUtils::replace_expr(view_column_list, new_column_list, view_select))) { - LOG_WARN("failed to replace expr", K(ret)); - } else if (OB_FAIL(new_select_list.push_back(view_select))) { + } else if (OB_FAIL(old_column_exprs.push_back(col_expr))) { LOG_WARN("failed to push back expr", K(ret)); - } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(view_select, aggr_items))) { - LOG_WARN("failed to extract aggr expr", K(ret)); - } else if (OB_FAIL(append(temp_table_query->get_aggr_items(), aggr_items))) { - LOG_WARN("failed to append aggr items", K(ret)); - } else if (OB_FAIL(ObTransformUtils::extract_winfun_expr(view_select, win_func_exprs))) { - LOG_WARN("failed to extract win func exprs", K(ret)); - } else if (OB_FAIL(append(temp_table_query->get_window_func_exprs(), win_func_exprs))) { - LOG_WARN("failed to append win func exprs", K(ret)); + } + //select item是否存在于temp table中 + for (int64_t j = 0; OB_SUCC(ret) && !find && j < temp_table_select_list.count(); ++j) { + ObRawExpr *temp_table_select = temp_table_select_list.at(j); + if (OB_ISNULL(temp_table_select)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null select expr", K(ret)); + } else if (!temp_table_select->same_as(*view_select, &context)) { + //do nothing + } else if (OB_FAIL(new_select_list.push_back(temp_table_select))) { + LOG_WARN("failed to push back expr", K(ret)); + } else { + find = true; + } + } + //不存在于temp table中的select expr需要转换成temp table的select item + if (OB_SUCC(ret) && !find) { + ObSEArray aggr_items; + ObSEArray win_func_exprs; + if (OB_FAIL(ObTransformUtils::replace_expr(view_column_list, new_column_list, view_select))) { + LOG_WARN("failed to replace expr", K(ret)); + } else if (OB_FAIL(new_select_list.push_back(view_select))) { + LOG_WARN("failed to push back expr", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(view_select, aggr_items))) { + LOG_WARN("failed to extract aggr expr", K(ret)); + } else if (OB_FAIL(append(temp_table_query->get_aggr_items(), aggr_items))) { + LOG_WARN("failed to append aggr items", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_winfun_expr(view_select, win_func_exprs))) { + LOG_WARN("failed to extract win func exprs", K(ret)); + } else if (OB_FAIL(append(temp_table_query->get_window_func_exprs(), win_func_exprs))) { + LOG_WARN("failed to append win func exprs", K(ret)); + } } } - } - //为temp table创建新的select item，并替换parent stmt的引用 - if (OB_SUCC(ret)) { - ObSEArray new_column_exprs; - view_table->ref_query_ = temp_table_query; - if (OB_FALSE_IT(parent_stmt->clear_column_items())) { - } else if (OB_FAIL(ObTransformUtils::create_columns_for_view(ctx_, - *view_table, - parent_stmt, - new_select_list, - new_column_exprs))) { - LOG_WARN("failed to create column for view", K(ret)); - } else if (OB_FAIL(parent_stmt->replace_relation_exprs(old_column_exprs, new_column_exprs))) { - LOG_WARN("failed to replace inner stmt expr", K(ret)); - } else if (OB_FAIL(temp_table_query->adjust_subquery_list())) { - LOG_WARN("failed to adjust subquery list", K(ret)); + //为temp table创建新的select item，并替换parent stmt的引用 + if (OB_SUCC(ret)) { + ObSEArray new_column_exprs; + view_table->ref_query_ = temp_table_query; + if (OB_FALSE_IT(parent_stmt->clear_column_items())) { + } else if (OB_FAIL(ObTransformUtils::create_columns_for_view(ctx_, + *view_table, + parent_stmt, + new_select_list, + new_column_exprs))) { + LOG_WARN("failed to create column for view", K(ret)); + } else if (OB_FAIL(parent_stmt->replace_relation_exprs(old_column_exprs, new_column_exprs))) { + LOG_WARN("failed to replace inner stmt expr", K(ret)); + } else if (OB_FAIL(temp_table_query->adjust_subquery_list())) { + LOG_WARN("failed to adjust subquery list", K(ret)); + } } - } + } // end smart var return ret; } diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 50fbecaaa..55723bf51 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -34,6 +34,10 @@ #include "sql/rewrite/ob_equal_analysis.h" #include "sql/rewrite/ob_transform_utils.h" #include "sql/optimizer/ob_optimizer_util.h" +#include "sql/resolver/mv/ob_mv_provider.h" +#include "sql/resolver/dml/ob_select_resolver.h" +#include "sql/parser/ob_parser.h" +#include "sql/rewrite/ob_transform_pre_process.h" namespace oceanbase { using namespace common; @@ -1712,7 +1716,7 @@ int ObTransformUtils::flatten_joined_table(ObDMLStmt *stmt) LOG_WARN("failed to push back joined table", K(ret)); } else if (i < origin_joined_table_count) { /* do nohing */ - } else if (stmt->add_from_item(table->table_id_, true)) { + } else if (OB_FAIL(stmt->add_from_item(table->table_id_, true))) { LOG_WARN("failed to add from item", K(ret), K(right_table)); } } else if (OB_FAIL(append(stmt->get_condition_exprs(), table->get_join_conditions()))) { @@ -1723,13 +1727,13 @@ int ObTransformUtils::flatten_joined_table(ObDMLStmt *stmt) } else { if (left_table->is_joined_table()) { ret = tmp_joined_tables.push_back(static_cast(left_table)); - } else if (stmt->add_from_item(left_table->table_id_, false)) { + } else if (OB_FAIL(stmt->add_from_item(left_table->table_id_, false))) { LOG_WARN("failed to add from item", K(ret), K(left_table)); } if (OB_FAIL(ret)) { } else if (right_table->is_joined_table()) { ret = tmp_joined_tables.push_back(static_cast(right_table)); - } else if (stmt->add_from_item(right_table->table_id_, false)) { + } else if (OB_FAIL(stmt->add_from_item(right_table->table_id_, false))) { LOG_WARN("failed to add from item", K(ret), K(right_table)); } } @@ -10881,7 +10885,8 @@ int ObTransformUtils::add_param_bool_constraint(ObTransformerCtx *ctx, int ObTransformUtils::get_all_child_stmts(ObDMLStmt *stmt, ObIArray &child_stmts, - hash::ObHashMap *parent_map) + hash::ObHashMap *parent_map, + const ObIArray *ignore_stmts /* default null */) { int ret = OB_SUCCESS; ObSEArray temp_stmts; @@ -10907,8 +10912,17 @@ int ObTransformUtils::get_all_child_stmts(ObDMLStmt *stmt, } } if (OB_SUCC(ret)) { - if (OB_FAIL(append(child_stmts, temp_stmts))) { - LOG_WARN("failed to append temp stmts", K(ret)); + if (NULL == ignore_stmts) { + if (OB_FAIL(append(child_stmts, temp_stmts))) { + LOG_WARN("failed to append temp stmts", K(ret)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret)&& i < temp_stmts.count(); ++i) { + if (!ObOptimizerUtil::find_item(*ignore_stmts, temp_stmts.at(i)) + && OB_FAIL(child_stmts.push_back(temp_stmts.at(i)))) { + LOG_WARN("failed to push back stmt", K(ret)); + } + } } } for (int64_t i = 0; OB_SUCC(ret)&& i < temp_stmts.count(); ++i) { @@ -10924,9 +10938,11 @@ int ObTransformUtils::get_all_child_stmts(ObDMLStmt *stmt, if (OB_ISNULL(temp_stmts.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("temp stmt is null", K(ret)); + } else if (NULL != ignore_stmts && ObOptimizerUtil::find_item(*ignore_stmts, temp_stmts.at(i))) { + /* do nothing */ } else if (parent_map != NULL && OB_FAIL(parent_map->set_refactored(key, parent_stmt))) { LOG_WARN("failed to add parent child relation", K(ret)); - } else if (OB_FAIL(SMART_CALL(get_all_child_stmts(temp_stmts.at(i), child_stmts, parent_map)))) { + } else if (OB_FAIL(SMART_CALL(get_all_child_stmts(temp_stmts.at(i), child_stmts, parent_map, ignore_stmts)))) { LOG_WARN("failed to get all child stmts", K(ret)); } } @@ -14331,6 +14347,161 @@ int ObTransformUtils::expand_temp_table(ObTransformerCtx *ctx, ObDMLStmt::TempTa return ret; } +int ObTransformUtils::expand_mview_table(ObTransformerCtx *ctx, ObDMLStmt *upper_stmt, TableItem *rt_mv_table) +{ + int ret = OB_SUCCESS; + const ObTableSchema *mv_schema = NULL; + if (OB_ISNULL(ctx) || OB_ISNULL(upper_stmt) || OB_ISNULL(rt_mv_table) + || OB_ISNULL(ctx->allocator_) + || OB_UNLIKELY(!rt_mv_table->need_expand_rt_mv_) + || OB_UNLIKELY(!ctx->is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null", K(ret), K(ctx), KPC(rt_mv_table)); + } else { + const uint64_t OB_MAX_SUBQUERY_NAME_LENGTH = 64; + const char *EXPAND_VIEW_PREFIX = "RT_"; + ObString expand_view; + ObString expand_view_name; + int64_t pos = 0; + char buf[OB_MAX_SUBQUERY_NAME_LENGTH]; + int64_t buf_len = OB_MAX_SUBQUERY_NAME_LENGTH; + ObSelectStmt *view_stmt = NULL; + OPT_TRACE("expand real time materialized view: ", rt_mv_table->get_object_name()); + OPT_TRACE_BEGIN_SECTION; + ObMVProvider mv_provider(ctx->session_info_->get_effective_tenant_id(), rt_mv_table->mview_id_, true); + if (OB_FAIL(mv_provider.init_mv_provider(ctx->sql_schema_guard_->get_schema_guard(), + ctx->session_info_))) { + LOG_WARN("fail to init mv provider", K(ret)); + } else if (OB_FAIL(mv_provider.get_real_time_mv_expand_view(*ctx->allocator_, expand_view))) { + LOG_WARN("fail to get real time mv expand view", K(ret)); + } else if (OB_FAIL(generate_view_stmt_from_query_string(expand_view, ctx, view_stmt))) { + LOG_WARN("fail to genearte real time mview stmt", K(ret), K(expand_view)); + } else if (OB_FAIL(ctx->temp_table_ignore_stmts_.push_back(view_stmt))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(set_expand_mview_flag(view_stmt))) { + LOG_WARN("fail to set expand mview flag", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("%s", EXPAND_VIEW_PREFIX))) { + LOG_WARN("append expand view prefix to buf error", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("%.*s", rt_mv_table->get_object_name().length(), rt_mv_table->get_object_name().ptr()))) { + LOG_WARN("append mv table name to buf error", K(ret)); + } else if (OB_FALSE_IT(expand_view_name = common::ObString::make_string(buf))) { + } else if (OB_FAIL(ob_write_string(*ctx->allocator_, + expand_view_name, + rt_mv_table->table_name_))) { + LOG_WARN("failed to write string", K(ret)); + } else { + rt_mv_table->type_ = TableItem::GENERATED_TABLE; + rt_mv_table->ref_id_ = OB_INVALID_ID; + rt_mv_table->mview_id_ = OB_INVALID_ID; + rt_mv_table->database_name_ = ObString::make_string(""); + rt_mv_table->ref_query_ = view_stmt; + rt_mv_table->table_type_ = MAX_TABLE_TYPE; + rt_mv_table->need_expand_rt_mv_ = false; + const ObIArray &col_items = upper_stmt->get_column_items(); + ObIArray &sel_items = view_stmt->get_select_items(); + int64_t pos = OB_INVALID_ID; + // keep the result type for expand real-time view select items + for (int64_t i = 0; OB_SUCC(ret) && i < col_items.count(); ++i) { + if (OB_ISNULL(col_items.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (col_items.at(i).table_id_ != rt_mv_table->table_id_) { + /* do nothing */ + } else if (FALSE_IT(pos = col_items.at(i).expr_->get_column_id() - OB_APP_MIN_COLUMN_ID)) { + } else if (OB_UNLIKELY(pos < 0 || pos >= sel_items.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid array pos", K(pos), K(sel_items.count()), K(ret)); + } else if (OB_FAIL(ObTransformUtils::add_cast_for_replace_if_need(*ctx->expr_factory_, + col_items.at(i).expr_, + sel_items.at(pos).expr_, + ctx->session_info_))) { + LOG_WARN("try add cast expr above failed", K(ret)); + } else { + col_items.at(i).expr_->set_is_rowkey_column(false); + } + } + } + OPT_TRACE_END_SECTION; + } + return ret; +} + +// 1. 不进行参数化 +// 2. 不需要检查权限 +// 3. 添加 dependency table +// 4. parser 特殊路径处理: batch multi stmt 禁止，values table +int ObTransformUtils::generate_view_stmt_from_query_string(const ObString &query_str, + ObTransformerCtx *ctx, + ObSelectStmt *&view_stmt) +{ + int ret = OB_SUCCESS; + view_stmt = NULL; + ParseResult parse_result; + ParseNode *node = NULL; + ObParser parser(*ctx->allocator_, ctx->session_info_->get_sql_mode(), ctx->session_info_->get_charsets4parser()); + ObResolverParams resolver_ctx; + resolver_ctx.is_for_rt_mv_ = true; + resolver_ctx.allocator_ = ctx->allocator_; + resolver_ctx.schema_checker_ = ctx->schema_checker_; + resolver_ctx.session_info_ = ctx->session_info_; + resolver_ctx.expr_factory_ = ctx->expr_factory_; + resolver_ctx.stmt_factory_ = ctx->stmt_factory_; + resolver_ctx.sql_proxy_ = GCTX.sql_proxy_; + resolver_ctx.query_ctx_ = ctx->stmt_factory_->get_query_ctx(); + ObSelectResolver select_resolver(resolver_ctx); + ObTransformPreProcess trans(ctx); + trans.set_transformer_type(PRE_PROCESS); + uint64_t dummy_value = 0; + ObDMLStmt *dml_stmt = NULL; + ObQueryHint &query_hint = ctx->stmt_factory_->get_query_ctx()->get_query_hint_for_update(); + const int64_t stmt_count = query_hint.stmt_id_map_.count(); + if (OB_ISNULL(resolver_ctx.query_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(resolver_ctx.query_ctx_)); + } else if (OB_FAIL(parser.parse(query_str, parse_result))) { + LOG_WARN("parse view definition failed", K(ret), K(query_str)); + } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || OB_UNLIKELY(T_SELECT != node->type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv select node", K(ret), K(node), K(node->type_)); + } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ += static_cast(parse_result.question_mark_ctx_.count_))) { + } else if (OB_FAIL(select_resolver.resolve(*node))) { + LOG_WARN("resolve view definition failed", K(ret)); + } else if (OB_ISNULL(dml_stmt = static_cast(select_resolver.get_basic_stmt()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv stmt", K(ret), K(dml_stmt)); + } else if (OB_FAIL(query_hint.generate_orig_stmt_qb_name(*ctx->allocator_, stmt_count))) { + LOG_WARN("failed to generate stmt name after resolve", K(ret)); + } else if (OB_FAIL(query_hint.distribute_hint_to_orig_stmt(dml_stmt))) { + LOG_WARN("faild to distribute hint to orig stmt", K(ret)); + } else if (OB_FAIL(trans.transform(dml_stmt, dummy_value))) { + LOG_WARN("failed to do transform pre processing", K(ret)); + } else { + view_stmt = static_cast(dml_stmt); + LOG_DEBUG("generate mv stmt", KPC(view_stmt)); + } + return ret; +} + +int ObTransformUtils::set_expand_mview_flag(ObSelectStmt *view_stmt) +{ + int ret = OB_SUCCESS; + ObSEArray child_stmts; + if (OB_ISNULL(view_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("view_stmt is null", K(ret)); + } else if (OB_FAIL(view_stmt->get_child_stmts(child_stmts))) { + LOG_WARN("failed to get child stmts", K(ret)); + } else { + view_stmt->set_expanded_mview(true); + for (int64_t i = 0; OB_SUCC(ret) && i < child_stmts.count(); ++i) { + if (OB_FAIL(SMART_CALL(set_expand_mview_flag(child_stmts.at(i))))) { + LOG_WARN("failed to set child expand mview flag", K(ret), K(i)); + } + } + } + return ret; +} + ObSelectStmtPointer::ObSelectStmtPointer() : stmt_group_() { } @@ -14863,6 +15034,33 @@ int ObTransformUtils::check_child_projection_validity(const ObSelectStmt *child_ return ret; } +int ObTransformUtils::add_aggr_winfun_expr(ObSelectStmt *stmt, + ObRawExpr *expr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr) || OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(expr), K(stmt)); + } else if (expr->is_aggr_expr()) { + ObAggFunRawExpr *agg_expr = static_cast(expr); + if (OB_FAIL(stmt->add_agg_item(*agg_expr))) { + LOG_WARN("failed to add agg item", K(ret), KPC(agg_expr)); + } + } else if (expr->is_win_func_expr()) { + ObWinFunRawExpr *win_expr = static_cast(expr); + if (OB_FAIL(stmt->get_window_func_exprs().push_back(win_expr))) { + LOG_WARN("failed to add win func expr", K(ret), KPC(win_expr)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { + if (OB_FAIL(SMART_CALL(add_aggr_winfun_expr(stmt, expr->get_param_expr(i))))) { + LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(expr->get_param_expr(i))); + } + } + } + return ret; +} + //check whether the filter can be used as a partition topn filter for winfunc_exprs int ObTransformUtils::is_winfunc_topn_filter(const ObIArray &winfunc_exprs, ObRawExpr *filter, diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index 194a8861f..a03e3e224 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -1511,7 +1511,8 @@ public: static int get_all_child_stmts(ObDMLStmt *stmt, ObIArray &child_stmts, - hash::ObHashMap *parent_map = NULL); + hash::ObHashMap *parent_map = NULL, + const ObIArray *ignore_stmts = NULL); static int check_select_expr_is_const(ObSelectStmt *stmt, ObRawExpr *expr, bool &is_const); @@ -1883,6 +1884,15 @@ public: static bool is_const_null(ObRawExpr &expr); static bool is_full_group_by(ObSelectStmt& stmt, ObSQLMode mode); + static int add_aggr_winfun_expr(ObSelectStmt *stmt, + ObRawExpr *expr); + static int expand_mview_table(ObTransformerCtx *ctx, ObDMLStmt *upper_stmt, TableItem *rt_mv_table); + + static int generate_view_stmt_from_query_string(const ObString &expand_view, + ObTransformerCtx *ctx, + ObSelectStmt *&view_stmt); + static int set_expand_mview_flag(ObSelectStmt *view_stmt); + static int is_where_subquery_correlated(const ObIArray &exec_params, const ObSelectStmt &subquery, bool &is_correlated); @@ -1920,7 +1930,6 @@ public: bool &is_ref); static int check_contain_correlated_lateral_table(ObDMLStmt *stmt, bool &is_contain); - private: static int inner_get_lazy_left_join(ObDMLStmt *stmt, TableItem *table, diff --git a/src/sql/rewrite/ob_transform_win_magic.cpp b/src/sql/rewrite/ob_transform_win_magic.cpp index 229f04cf6..467e4e15a 100644 --- a/src/sql/rewrite/ob_transform_win_magic.cpp +++ b/src/sql/rewrite/ob_transform_win_magic.cpp @@ -41,20 +41,20 @@ int ObTransformWinMagic::transform_one_stmt(common::ObIArray &p int ret = OB_SUCCESS; int64_t drill_down_idx = -1; int64_t roll_up_idx = -2; // -1 means main stmt - ObStmtCompareContext context; - ObStmtMapInfo map_info; ObSEArray trans_tables; - if (OB_ISNULL(stmt) || OB_ISNULL(stmt->get_query_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("stmt is null", K(ret), K(stmt)); - } else if (OB_FAIL(get_view_to_trans(stmt, drill_down_idx, roll_up_idx, context, map_info, trans_tables))) { - LOG_WARN("get view to trans failed", K(ret)); - } else if (drill_down_idx == -1 || roll_up_idx == -2) { - // no valid item to do trans - } else if (OB_FAIL(do_transform(parent_stmts, stmt, drill_down_idx, - roll_up_idx, map_info, context, trans_happened, trans_tables))) { - LOG_WARN("do win magic transform for from failed", K(ret)); - } + SMART_VARS_2((ObStmtCompareContext, context), (ObStmtMapInfo, map_info)) { + if (OB_ISNULL(stmt) || OB_ISNULL(stmt->get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stmt is null", K(ret), K(stmt)); + } else if (OB_FAIL(get_view_to_trans(stmt, drill_down_idx, roll_up_idx, context, map_info, trans_tables))) { + LOG_WARN("get view to trans failed", K(ret)); + } else if (drill_down_idx == -1 || roll_up_idx == -2) { + // no valid item to do trans + } else if (OB_FAIL(do_transform( + parent_stmts, stmt, drill_down_idx, roll_up_idx, map_info, context, trans_happened, trans_tables))) { + LOG_WARN("do win magic transform for from failed", K(ret)); + } + } // end smart vars return ret; } @@ -713,6 +713,7 @@ int ObTransformWinMagic::check_stmt_and_view(ObDMLStmt *stmt, EqualSets dummy_set; ObSelectStmt *rewrite_view = NULL; int64_t match_count = 0; + QueryRelation relation; if (OB_ISNULL(stmt) || OB_ISNULL(rewrite_table) || OB_ISNULL(rewrite_view = rewrite_table->ref_query_)) { ret = OB_ERR_UNEXPECTED; @@ -764,7 +765,7 @@ int ObTransformWinMagic::check_stmt_and_view(ObDMLStmt *stmt, if (OB_FAIL(ret) || !is_valid) { //do nothing } else if (OB_FAIL(ObStmtComparer::compute_semi_infos_map(rewrite_view, stmt, - map_info, match_count))) { + true, map_info, match_count))) { LOG_WARN("failed to compute semi info map", K(ret)); } else if (match_count != rewrite_view->get_semi_info_size() || match_count != stmt->get_semi_info_size()) { @@ -774,7 +775,7 @@ int ObTransformWinMagic::check_stmt_and_view(ObDMLStmt *stmt, LOG_WARN("get column exprs failed", K(ret)); } else if (OB_FAIL(ObStmtComparer::compute_conditions_map(rewrite_view, stmt, rewrite_view->get_group_exprs(), - column_exprs, map_info, expr_map, match_count))) { + column_exprs, map_info, expr_map, relation))) { LOG_WARN("compute expr map failed", K(ret)); } else if (OB_FAIL(ObOptimizerUtil::get_parent_stmt_exprs(dummy_set, rewrite_table->table_id_, *stmt, *rewrite_view, @@ -817,6 +818,7 @@ int ObTransformWinMagic::check_view_and_view(ObDMLStmt *main_stmt, ObSEArray reverse_map; ObSqlBitSet<> matched_cond; int64_t match_count = 0; + QueryRelation relation; EqualSets dummy_set; if (OB_ISNULL(main_stmt) || OB_ISNULL(drill_down_table) || OB_ISNULL(roll_up_table) || OB_ISNULL(drill_down_view = drill_down_table->ref_query_) @@ -835,7 +837,7 @@ int ObTransformWinMagic::check_view_and_view(ObDMLStmt *main_stmt, } else if (!is_valid) { //do nothing } else if (OB_FAIL(ObStmtComparer::compute_semi_infos_map(drill_down_view, roll_up_view, - map_info, match_count))) { + true, map_info, match_count))) { LOG_WARN("failed to compute semi info map", K(ret)); } else if (match_count != drill_down_view->get_semi_info_size() || match_count != roll_up_view->get_semi_info_size()) { @@ -843,7 +845,7 @@ int ObTransformWinMagic::check_view_and_view(ObDMLStmt *main_stmt, } else if (OB_FAIL(ObStmtComparer::compute_conditions_map(drill_down_view, roll_up_view, drill_down_view->get_group_exprs(), roll_up_view->get_group_exprs(), map_info, - map_info.group_map_, match_count))) { + map_info.group_map_, relation))) { LOG_WARN("compute expr map failed", K(ret)); } else if (OB_FAIL(get_reverse_map(map_info.group_map_, reverse_map, roll_up_view->get_group_exprs().count()))) { LOG_WARN("get reverse map failed", K(ret)); diff --git a/src/sql/rewrite/ob_transformer_impl.cpp b/src/sql/rewrite/ob_transformer_impl.cpp index 81bf01e3e..9f2c61641 100644 --- a/src/sql/rewrite/ob_transformer_impl.cpp +++ b/src/sql/rewrite/ob_transformer_impl.cpp @@ -47,6 +47,7 @@ #include "sql/rewrite/ob_transform_count_to_exists.h" #include "sql/rewrite/ob_transform_expr_pullup.h" #include "sql/rewrite/ob_transform_dblink.h" +#include "sql/rewrite/ob_transform_mv_rewrite.h" #include "sql/rewrite/ob_transform_decorrelate.h" #include "common/ob_smart_call.h" #include "sql/engine/ob_exec_context.h" @@ -295,6 +296,7 @@ int ObTransformerImpl::transform_rule_set(ObDMLStmt *&stmt, int64_t i = 0; for (i = 0; OB_SUCC(ret) && need_next_iteration && i < iteration_count; ++i) { bool trans_happened_in_iteration = false; + ctx_->iteration_level_ = i; LOG_TRACE("start to transform one iteration", K(i)); OPT_TRACE("-- begin ", i, " iteration"); if (OB_FAIL(transform_rule_set_in_one_iteration(stmt, @@ -336,6 +338,7 @@ int ObTransformerImpl::transform_rule_set_in_one_iteration(ObDMLStmt *&stmt, * The ordering to apply the following rules is important, * think carefully when new rules are added */ + APPLY_RULE_IF_NEEDED(MV_REWRITE, ObTransformMVRewrite); APPLY_RULE_IF_NEEDED(SIMPLIFY_EXPR, ObTransformSimplifyExpr); APPLY_RULE_IF_NEEDED(SIMPLIFY_DISTINCT, ObTransformSimplifyDistinct); APPLY_RULE_IF_NEEDED(SIMPLIFY_GROUPBY, ObTransformSimplifyGroupby); @@ -502,7 +505,7 @@ int ObTransformerImpl::check_temp_table_functions(ObDMLStmt *stmt, StmtFunc &fun return ret; } -int ObTransformerImpl::check_stmt_functions(ObDMLStmt *stmt, StmtFunc &func) +int ObTransformerImpl::check_stmt_functions(const ObDMLStmt *stmt, StmtFunc &func) { int ret = OB_SUCCESS; if (OB_ISNULL(stmt)) { @@ -527,7 +530,7 @@ int ObTransformerImpl::check_stmt_functions(ObDMLStmt *stmt, StmtFunc &func) } for (int64_t i = 0; OB_SUCC(ret) && !func.contain_link_table_ && i < stmt->get_table_items().count(); ++i) { - TableItem *table = stmt->get_table_item(i); + const TableItem *table = stmt->get_table_item(i); if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null table item", K(ret)); @@ -537,7 +540,7 @@ int ObTransformerImpl::check_stmt_functions(ObDMLStmt *stmt, StmtFunc &func) } for (int64_t i = 0; OB_SUCC(ret) && !func.contain_json_table_ && i < stmt->get_table_items().count(); ++i) { - TableItem *table = stmt->get_table_item(i); + const TableItem *table = stmt->get_table_item(i); if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null table item", K(ret)); @@ -554,7 +557,7 @@ int ObTransformerImpl::check_stmt_functions(ObDMLStmt *stmt, StmtFunc &func) stmt->is_update_stmt() || stmt->is_merge_stmt() || stmt->is_insert_stmt())) { - ObDelUpdStmt *del_upd_stmt = static_cast(stmt); + const ObDelUpdStmt *del_upd_stmt = static_cast(stmt); func.update_global_index_ = func.update_global_index_ || del_upd_stmt->has_global_index(); } if (OB_SUCC(ret) && !func.all_found()) { diff --git a/src/sql/rewrite/ob_transformer_impl.h b/src/sql/rewrite/ob_transformer_impl.h index 4df18f6ca..2dc2b3374 100644 --- a/src/sql/rewrite/ob_transformer_impl.h +++ b/src/sql/rewrite/ob_transformer_impl.h @@ -143,7 +143,7 @@ public: bool contain_link_table_; bool contain_json_table_; }; - int check_stmt_functions(ObDMLStmt *stmt, StmtFunc &func); + static int check_stmt_functions(const ObDMLStmt *stmt, StmtFunc &func); int check_temp_table_functions(ObDMLStmt *stmt, StmtFunc &func); inline ObTransformerCtx *get_trans_ctx() { return ctx_; } private: @@ -190,21 +190,22 @@ int ObTransformerImpl::transform_one_rule(ObDMLStmt *&stmt, LOG_WARN("unexpect null param", K(ret)); } else if (is_type_needed(needed_types & needed_transform_types_, type)) { - T trans(ctx_); - trans.set_transformer_type(type); - OPT_TRACE_TITLE("start transform rule", rule_name); - if (OB_FAIL(THIS_WORKER.check_status())) { - LOG_WARN("check status fail", K(ret)); - } else if (OB_FAIL(trans.transform(stmt, needed_transform_types_))) { - LOG_WARN("failed to transform a rewrite rule", "class", rule_name, K(ret), K(ctx_->outline_trans_hints_)); - } else if (OB_FAIL(collect_trans_stat(trans))) { - LOG_WARN("failed to collect transform stat", K(ret)); - } else { - trans_happened |= trans.get_trans_happened(); - OPT_TRACE_TIME_USED; - OPT_TRACE_MEM_USED; - LOG_TRACE("succeed to transform a rewrite rule", "class", - rule_name, K(trans.get_trans_happened()), K(ret)); + SMART_VAR(T, trans, ctx_) { + trans.set_transformer_type(type); + OPT_TRACE_TITLE("start transform rule", rule_name); + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status fail", K(ret)); + } else if (OB_FAIL(trans.transform(stmt, needed_transform_types_))) { + LOG_WARN("failed to transform a rewrite rule", "class", rule_name, K(ret), K(ctx_->outline_trans_hints_)); + } else if (OB_FAIL(collect_trans_stat(trans))) { + LOG_WARN("failed to collect transform stat", K(ret)); + } else { + trans_happened |= trans.get_trans_happened(); + OPT_TRACE_TIME_USED; + OPT_TRACE_MEM_USED; + LOG_TRACE("succeed to transform a rewrite rule", "class", + rule_name, K(trans.get_trans_happened()), K(ret)); + } } } else { LOG_TRACE("skip tranform a rewrite rule", "class", rule_name); diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 6954774b9..53672c850 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -3554,6 +3554,16 @@ int ObBasicSessionInfo::is_transformation_enabled(bool &transformation_enabled) return get_bool_sys_var(SYS_VAR_OB_ENABLE_TRANSFORMATION, transformation_enabled); } +int ObBasicSessionInfo::get_query_rewrite_enabled(int64_t &query_rewrite_enabled) const +{ + return get_int64_sys_var(SYS_VAR_QUERY_REWRITE_ENABLED, query_rewrite_enabled); +} + +int ObBasicSessionInfo::get_query_rewrite_integrity(int64_t &query_rewrite_integrity) const +{ + return get_int64_sys_var(SYS_VAR_QUERY_REWRITE_INTEGRITY, query_rewrite_integrity); +} + int ObBasicSessionInfo::is_serial_set_order_forced(bool &force_set_order, bool is_oracle_mode) const { int ret = OB_SUCCESS; diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 3bbcfc39f..b0be28f40 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -1024,6 +1024,8 @@ public: // 以下helper函数是为了方便查看某系统变量的值 int if_aggr_pushdown_allowed(bool &aggr_pushdown_allowed) const; int is_transformation_enabled(bool &transformation_enabled) const; + int get_query_rewrite_enabled(int64_t &query_rewrite_enabled) const; + int get_query_rewrite_integrity(int64_t &query_rewrite_integrity) const; int is_serial_set_order_forced(bool &force_set_order, bool is_oracle_mode) const; int is_storage_estimation_enabled(bool &storage_estimation_enabled) const; bool is_use_trace_log() const diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index db8f5a168..178183ae0 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -1216,6 +1216,7 @@ public: int is_preserve_order_for_pagination_enabled(bool &enabled) const; ObSessionDDLInfo &get_ddl_info() { return ddl_info_; } + const ObSessionDDLInfo &get_ddl_info() const { return ddl_info_; } void set_ddl_info(const ObSessionDDLInfo &ddl_info) { ddl_info_ = ddl_info; } bool is_table_name_hidden() const { return is_table_name_hidden_; } void set_table_name_hidden(const bool is_hidden) { is_table_name_hidden_ = is_hidden; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 2bbfefdaf..ec80db5dc 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -759,6 +759,7 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_sstable_scan_merge.cpp direct_load/ob_direct_load_sstable_scan_merge_loser_tree.cpp direct_load/ob_direct_load_sstable_scanner.cpp + direct_load/ob_direct_load_struct.cpp direct_load/ob_direct_load_table_data_desc.cpp direct_load/ob_direct_load_table_store.cpp direct_load/ob_direct_load_tmp_file.cpp diff --git a/src/storage/ddl/ob_ddl_server_client.cpp b/src/storage/ddl/ob_ddl_server_client.cpp index 037ca7ab6..2e1ee2f3d 100644 --- a/src/storage/ddl/ob_ddl_server_client.cpp +++ b/src/storage/ddl/ob_ddl_server_client.cpp @@ -281,7 +281,7 @@ int ObDDLServerClient::abort_redef_table(const obrpc::ObAbortRedefTableArg &arg, const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); int64_t MAX_ABORT_WAIT_TIMEOUT = 60 * 1000 * 1000; //60s THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + MAX_ABORT_WAIT_TIMEOUT); - if (OB_FAIL(sql::ObDDLExecutorUtil::wait_ddl_finish(arg.tenant_id_, arg.task_id_, session, common_rpc_proxy))) { + if (OB_FAIL(sql::ObDDLExecutorUtil::wait_ddl_finish(arg.tenant_id_, arg.task_id_, DDL_DIRECT_LOAD, session, common_rpc_proxy))) { if (OB_CANCELED == ret) { ret = OB_SUCCESS; LOG_INFO("ddl abort success", K_(arg.task_id)); @@ -356,7 +356,7 @@ int ObDDLServerClient::finish_redef_table(const obrpc::ObFinishRedefTableArg &fi if (OB_FAIL(ret)) { } else if (OB_FAIL(build_ddl_single_replica_response(build_single_arg))) { LOG_WARN("build ddl single replica response", K(ret), K(build_single_arg)); - } else if (OB_FAIL(sql::ObDDLExecutorUtil::wait_ddl_finish(finish_redef_arg.tenant_id_, finish_redef_arg.task_id_, &session, common_rpc_proxy))) { + } else if (OB_FAIL(sql::ObDDLExecutorUtil::wait_ddl_finish(finish_redef_arg.tenant_id_, finish_redef_arg.task_id_, DDL_DIRECT_LOAD, &session, common_rpc_proxy))) { LOG_WARN("failed to wait ddl finish", K(ret), K(finish_redef_arg.tenant_id_), K(finish_redef_arg.task_id_)); } if (OB_TMP_FAIL(heart_beat_clear(finish_redef_arg.task_id_, tenant_id))) { diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index 027c08f58..165e500ea 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -870,12 +870,18 @@ int ObTenantDirectLoadMgr::check_and_process_finished_tablet( } } if (OB_SUCC(ret) && nullptr != row_iter) { + ObDDLInsertRowIterator *ddl_row_iter = reinterpret_cast(row_iter); const ObDatumRow *row = nullptr; const bool skip_lob = true; while (OB_SUCC(ret)) { if (OB_FAIL(THIS_WORKER.check_status())) { LOG_WARN("check status failed", K(ret)); - } else if (OB_FAIL(static_cast(row_iter)->get_next_row(skip_lob, row))) { + } else { + if (ddl_row_iter != nullptr) { + ret = ddl_row_iter->get_next_row(skip_lob, row); + } else { + ret = row_iter->get_next_row(row); + } if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index b45e06d88..f7af0ba19 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -59,6 +59,7 @@ ObDirectLoadMergeParam::ObDirectLoadMergeParam() is_column_store_(false), online_opt_stat_gather_(false), px_mode_(false), + insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE), insert_table_ctx_(nullptr), dml_row_handler_(nullptr) { @@ -73,6 +74,7 @@ bool ObDirectLoadMergeParam::is_valid() const return OB_INVALID_ID != table_id_ && 0 < rowkey_column_num_ && 0 < store_column_count_ && snapshot_version_ > 0 && table_data_desc_.is_valid() && nullptr != datum_utils_ && nullptr != col_descs_ && nullptr != cmp_funcs_ && + ObDirectLoadInsertMode::is_type_valid(insert_mode_) && nullptr != insert_table_ctx_ && nullptr != dml_row_handler_; } @@ -210,6 +212,7 @@ int ObDirectLoadTabletMergeCtx::init(ObTableLoadTableCtx *ctx, origin_table_param.table_id_ = param.table_id_; origin_table_param.tablet_id_ = ls_partition_id.part_tablet_id_.tablet_id_; origin_table_param.ls_id_ = ls_partition_id.ls_id_; + origin_table_param.insert_mode_ = param.insert_mode_; if (OB_FAIL(origin_table_.init(origin_table_param))) { LOG_WARN("fail to init origin sstable", KR(ret)); } else { diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.h b/src/storage/direct_load/ob_direct_load_merge_ctx.h index 9fde5b8a2..2d8303918 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.h +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.h @@ -22,6 +22,7 @@ #include "share/table/ob_table_load_dml_stat.h" #include "share/table/ob_table_load_sql_statistics.h" #include "storage/direct_load/ob_direct_load_origin_table.h" +#include "storage/direct_load/ob_direct_load_struct.h" #include "storage/direct_load/ob_direct_load_table_data_desc.h" #include "storage/direct_load/ob_direct_load_fast_heap_table.h" #include "observer/table_load/ob_table_load_table_ctx.h" @@ -54,7 +55,9 @@ public: TO_STRING_KV(K_(table_id), K_(target_table_id), K_(rowkey_column_num), K_(store_column_count), K_(snapshot_version), K_(table_data_desc), KP_(datum_utils), KP_(col_descs), KP_(lob_column_cnt), KP_(cmp_funcs), K_(is_heap_table), K_(is_fast_heap_table), - K_(is_column_store), K_(online_opt_stat_gather), KP_(insert_table_ctx), + K_(is_column_store), K_(online_opt_stat_gather), + "insert_mode", ObDirectLoadInsertMode::get_type_string(insert_mode_), + KP_(insert_table_ctx), KP_(dml_row_handler)); public: uint64_t table_id_; @@ -73,6 +76,7 @@ public: bool is_column_store_; bool online_opt_stat_gather_; bool px_mode_; + ObDirectLoadInsertMode::Type insert_mode_; ObDirectLoadInsertTableContext *insert_table_ctx_; ObDirectLoadDMLRowHandler *dml_row_handler_; }; diff --git a/src/storage/direct_load/ob_direct_load_origin_table.cpp b/src/storage/direct_load/ob_direct_load_origin_table.cpp index 8a3788d8c..b1a420a43 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.cpp +++ b/src/storage/direct_load/ob_direct_load_origin_table.cpp @@ -31,7 +31,7 @@ using namespace share::schema; */ ObDirectLoadOriginTableCreateParam::ObDirectLoadOriginTableCreateParam() - : table_id_(OB_INVALID_ID) + : table_id_(OB_INVALID_ID), insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) { } @@ -41,7 +41,10 @@ ObDirectLoadOriginTableCreateParam::~ObDirectLoadOriginTableCreateParam() bool ObDirectLoadOriginTableCreateParam::is_valid() const { - return OB_INVALID_ID != table_id_ && tablet_id_.is_valid() && ls_id_.is_valid(); + return OB_INVALID_ID != table_id_ && + tablet_id_.is_valid() && + ls_id_.is_valid() && + ObDirectLoadInsertMode::is_type_valid(insert_mode_); } /** @@ -49,7 +52,7 @@ bool ObDirectLoadOriginTableCreateParam::is_valid() const */ ObDirectLoadOriginTableMeta::ObDirectLoadOriginTableMeta() - : table_id_(OB_INVALID_ID) + : table_id_(OB_INVALID_ID), insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) { } @@ -96,12 +99,13 @@ int ObDirectLoadOriginTable::init(const ObDirectLoadOriginTableCreateParam ¶ LOG_WARN("unexpected ls is nullptr", KR(ret)); } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle_))) { LOG_WARN("fail to get tablet", KR(ret), K(tablet_id)); - } else if (OB_FAIL(prepare_tables())) { + } else if (ObDirectLoadInsertMode::need_origin_data(param.insert_mode_) && OB_FAIL(prepare_tables())) { LOG_WARN("fail to prepare tables", KR(ret)); } else { meta_.ls_id_ = param.ls_id_; meta_.table_id_ = param.table_id_; meta_.tablet_id_ = param.tablet_id_; + meta_.insert_mode_ = param.insert_mode_; is_inited_ = true; } } @@ -217,7 +221,9 @@ int ObDirectLoadOriginTableScanner::init(ObDirectLoadOriginTable *origin_table, LOG_WARN("Invalid argument", KR(ret), KPC(origin_table), K(query_range)); } else { origin_table_ = origin_table; - if (OB_FAIL((init_table_access_param()))) { + if (!ObDirectLoadInsertMode::need_origin_data(origin_table_->get_meta().insert_mode_)) { + // do nothing + } else if (OB_FAIL((init_table_access_param()))) { LOG_WARN("fail to init query range", KR(ret)); } else if (OB_FAIL(init_table_access_ctx())) { LOG_WARN("fail to init table access param", KR(ret)); @@ -228,7 +234,8 @@ int ObDirectLoadOriginTableScanner::init(ObDirectLoadOriginTable *origin_table, LOG_WARN("fail to init multi merge", KR(ret)); } else if (OB_FAIL(scan_merge_.open(query_range))) { LOG_WARN("fail to open multi merge", KR(ret), K(query_range)); - } else { + } + if (OB_SUCC(ret)) { is_inited_ = true; } } @@ -327,6 +334,8 @@ int ObDirectLoadOriginTableScanner::get_next_row(const ObDatumRow *&datum_row) if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObDirectLoadOriginTableScanner not init", KR(ret), KP(this)); + } else if (!ObDirectLoadInsertMode::need_origin_data(origin_table_->get_meta().insert_mode_)) { + ret = OB_ITER_END; } else { ObDatumRow *result_row = nullptr; if (OB_FAIL(scan_merge_.get_next_row(result_row))) { diff --git a/src/storage/direct_load/ob_direct_load_origin_table.h b/src/storage/direct_load/ob_direct_load_origin_table.h index 921bdd2d2..a7a057a73 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.h +++ b/src/storage/direct_load/ob_direct_load_origin_table.h @@ -14,6 +14,7 @@ #include "share/schema/ob_table_dml_param.h" #include "storage/access/ob_multiple_scan_merge.h" #include "storage/access/ob_store_row_iterator.h" +#include "storage/direct_load/ob_direct_load_struct.h" namespace oceanbase { @@ -26,11 +27,15 @@ public: ObDirectLoadOriginTableCreateParam(); ~ObDirectLoadOriginTableCreateParam(); bool is_valid() const; - TO_STRING_KV(K_(table_id), K_(tablet_id), K_(ls_id)); + TO_STRING_KV(K_(table_id), + K_(tablet_id), + K_(ls_id), + "insert_mode", ObDirectLoadInsertMode::get_type_string(insert_mode_)); public: uint64_t table_id_; common::ObTabletID tablet_id_; share::ObLSID ls_id_; + ObDirectLoadInsertMode::Type insert_mode_; }; struct ObDirectLoadOriginTableMeta @@ -38,11 +43,15 @@ struct ObDirectLoadOriginTableMeta public: ObDirectLoadOriginTableMeta(); ~ObDirectLoadOriginTableMeta(); - TO_STRING_KV( K_(table_id), K_(tablet_id), K_(ls_id)); + TO_STRING_KV(K_(table_id), + K_(tablet_id), + K_(ls_id), + "insert_mode", ObDirectLoadInsertMode::get_type_string(insert_mode_)); public: uint64_t table_id_; common::ObTabletID tablet_id_; share::ObLSID ls_id_; + ObDirectLoadInsertMode::Type insert_mode_; }; class ObDirectLoadOriginTable diff --git a/src/storage/direct_load/ob_direct_load_range_splitter.cpp b/src/storage/direct_load/ob_direct_load_range_splitter.cpp index 0884a36d7..b8b563b2d 100644 --- a/src/storage/direct_load/ob_direct_load_range_splitter.cpp +++ b/src/storage/direct_load/ob_direct_load_range_splitter.cpp @@ -208,7 +208,22 @@ int ObDirectLoadRangeSplitUtils::construct_origin_table_rowkey_iters( } else { const ObITableReadInfo &read_info = origin_table->get_tablet_handle().get_obj()->get_rowkey_read_info(); - if (nullptr != origin_table->get_major_sstable()) { + if (!ObDirectLoadInsertMode::need_origin_data(origin_table->get_meta().insert_mode_)) { + ObDirectLoadDatumRowkeyEmptyIterator *rowkey_iter = nullptr; + if (OB_ISNULL(rowkey_iter = OB_NEWx(ObDirectLoadDatumRowkeyEmptyIterator, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadDatumRowkeyEmptyIterator", KR(ret)); + } else if (OB_FAIL(rowkey_iters.push_back(rowkey_iter))) { + LOG_WARN("fail to push back rowkey iter", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != rowkey_iter) { + rowkey_iter->~ObDirectLoadDatumRowkeyEmptyIterator(); + allocator.free(rowkey_iter); + rowkey_iter = nullptr; + } + } + } else if (nullptr != origin_table->get_major_sstable()) { ObSSTable *major_sstable = origin_table->get_major_sstable(); ObIDirectLoadDatumRowkeyIterator *rowkey_iter = nullptr; if (OB_FAIL(ObDirectLoadRangeSplitUtils::construct_rowkey_iter( diff --git a/src/storage/direct_load/ob_direct_load_rowkey_iterator.h b/src/storage/direct_load/ob_direct_load_rowkey_iterator.h index 5c08af32a..4ba4e5574 100644 --- a/src/storage/direct_load/ob_direct_load_rowkey_iterator.h +++ b/src/storage/direct_load/ob_direct_load_rowkey_iterator.h @@ -34,9 +34,20 @@ public: TO_STRING_EMPTY(); }; +template +class ObDirectLoadRowkeyEmptyIterator : public ObIDirectLoadRowkeyIterator +{ +public: + ObDirectLoadRowkeyEmptyIterator() = default; + virtual ~ObDirectLoadRowkeyEmptyIterator() = default; + int get_next_rowkey(const Rowkey *&rowkey) override { return OB_ITER_END; } +}; + typedef ObIDirectLoadRowkeyIterator ObIDirectLoadDatumRowkeyIterator; typedef ObIDirectLoadRowkeyIterator ObIDirectLoadMultipleDatumRowkeyIterator; +typedef ObDirectLoadRowkeyEmptyIterator + ObDirectLoadDatumRowkeyEmptyIterator; class ObDirectLoadDatumRowkeyArrayIterator : public ObIDirectLoadDatumRowkeyIterator { diff --git a/src/storage/direct_load/ob_direct_load_struct.cpp b/src/storage/direct_load/ob_direct_load_struct.cpp new file mode 100644 index 000000000..d9ab008d1 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_struct.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_struct.h" + +namespace oceanbase +{ +namespace storage +{ + +/** + * ObDirectLoadMode + */ + +DEFINE_ENUM_FUNC(ObDirectLoadMode::Type, type, OB_DIRECT_LOAD_MODE_DEF, ObDirectLoadMode::); + +bool ObDirectLoadMode::is_type_valid(const Type type) +{ + return type > INVALID_MODE && type < MAX_MODE; +} + +/** + * ObDirectLoadMethod + */ + +DEFINE_ENUM_FUNC(ObDirectLoadMethod::Type, type, OB_DIRECT_LOAD_METHOD_DEF, ObDirectLoadMethod::); + +bool ObDirectLoadMethod::is_type_valid(const Type type) +{ + return type > INVALID_METHOD && type < MAX_METHOD; +} + +/** + * ObDirectLoadInsertMode + */ + +DEFINE_ENUM_FUNC(ObDirectLoadInsertMode::Type, type, OB_DIRECT_LOAD_INSERT_MODE_DEF, ObDirectLoadInsertMode::); + +bool ObDirectLoadInsertMode::is_type_valid(const Type type) +{ + return type > INVALID_INSERT_MODE && type < MAX_INSERT_MODE; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_struct.h b/src/storage/direct_load/ob_direct_load_struct.h new file mode 100644 index 000000000..ac80903b0 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_struct.h @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/string/ob_string.h" +#include "lib/utility/ob_macro_utils.h" +#include "lib/utility/ob_unify_serialize.h" + +namespace oceanbase +{ +namespace storage +{ + +struct ObDirectLoadMode +{ +#define OB_DIRECT_LOAD_MODE_DEF(DEF) \ + DEF(INVALID_MODE, = 0) \ + DEF(LOAD_DATA, = 1) \ + DEF(PDML, = 2) \ + DEF(TABLE_LOAD, = 3) \ + DEF(MAX_MODE, ) + + DECLARE_ENUM(Type, type, OB_DIRECT_LOAD_MODE_DEF, static); + + static bool is_type_valid(const Type type); +}; + +struct ObDirectLoadMethod +{ +#define OB_DIRECT_LOAD_METHOD_DEF(DEF) \ + DEF(INVALID_METHOD, = 0) \ + DEF(FULL, = 1) \ + DEF(INCREMENTAL, = 2) \ + DEF(MAX_METHOD, ) + + DECLARE_ENUM(Type, type, OB_DIRECT_LOAD_METHOD_DEF, static); + + static bool is_type_valid(const Type type); + static bool is_full(const Type type) { return FULL == type; } + static bool is_incremental(const Type type) { return INCREMENTAL == type; } +}; + +struct ObDirectLoadInsertMode +{ +#define OB_DIRECT_LOAD_INSERT_MODE_DEF(DEF) \ + DEF(INVALID_INSERT_MODE, = 0) \ + DEF(NORMAL, = 1) \ + DEF(INC_REPLACE, = 2) \ + DEF(OVERWRITE, = 3) \ + DEF(MAX_INSERT_MODE, ) + + DECLARE_ENUM(Type, type, OB_DIRECT_LOAD_INSERT_MODE_DEF, static); + + static bool is_type_valid(const Type type); + static bool is_valid_for_full_method(const Type type) { return NORMAL == type || OVERWRITE == type; } + static bool is_valid_for_incremental_method(const Type type) { return NORMAL == type || INC_REPLACE == type; } + static bool need_origin_data(const Type type) { return NORMAL == type; } +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/mview/ob_mview_refresh.cpp b/src/storage/mview/ob_mview_refresh.cpp index 975ae1215..15fd90398 100644 --- a/src/storage/mview/ob_mview_refresh.cpp +++ b/src/storage/mview/ob_mview_refresh.cpp @@ -250,7 +250,7 @@ int ObMViewRefresher::prepare_for_refresh() if (OB_SUCC(ret) && ObMVRefreshType::FAST == refresh_type) { const ObIArray *operators = nullptr; ObString fast_refresh_sql; - if (OB_FAIL(mv_provider.get_operators(operators))) { + if (OB_FAIL(mv_provider.get_fast_refresh_operators(operators))) { LOG_WARN("fail to get operators", KR(ret)); } else if (OB_ISNULL(operators)) { ret = OB_ERR_UNEXPECTED; @@ -435,7 +435,7 @@ int ObMViewRefresher::complete_refresh() .timeout(timeout_ctx.get_timeout()) .mview_complete_refresh(arg, res))) { LOG_WARN("fail to mview complete refresh", KR(ret), K(arg)); - } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, session_info, + } else if (OB_FAIL(ObDDLExecutorUtil::wait_ddl_finish(tenant_id, res.task_id_, DDL_MVIEW_COMPLETE_REFRESH, session_info, GCTX.rs_rpc_proxy_))) { LOG_WARN("fail to wait mview complete refresh finish", KR(ret)); } else { diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index d6151efae..e5c84d59a 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -559,6 +559,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -1144,6 +1145,9 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 3 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 3 | p_order | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 2 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 3 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -1994,6 +1998,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -2786,6 +2791,9 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 3 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 3 | p_order | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 2 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 3 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -3630,6 +3638,9 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_monitor_modified | 0 | oceanbase | PRIMARY | 3 | tablet_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 2 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_mview_dep | 0 | oceanbase | PRIMARY | 3 | p_order | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 2 | refresh_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_mview_refresh_change_stats | 0 | oceanbase | PRIMARY | 3 | mview_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index fdcf1c196..12425b69b 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -293,6 +293,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 505 __all_column_privilege 0 201001 1 506 __all_column_privilege_history 0 201001 1 507 __all_tenant_snapshot_ls_replica_history 0 201001 1 +518 __all_mview_dep 0 201001 1 10001 __tenant_virtual_all_table 2 201001 1 10002 __tenant_virtual_table_column 2 201001 1 10003 __tenant_virtual_table_index 2 201001 1 diff --git a/unittest/sql/resolver/expr/test_raw_expr_resolver.result b/unittest/sql/resolver/expr/test_raw_expr_resolver.result index ec0fe3d5b..df19f4e1a 100644 --- a/unittest/sql/resolver/expr/test_raw_expr_resolver.result +++ b/unittest/sql/resolver/expr/test_raw_expr_resolver.result @@ -1617,7 +1617,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 } [10] utc_timestamp() { @@ -1657,7 +1658,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 } [11] floor(3.14) { @@ -1726,7 +1728,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 } [12] count(*) { @@ -2404,7 +2407,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 } [19] @a { @@ -2477,7 +2481,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 } [20] c1 { @@ -5046,7 +5051,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 }, { "item_type":"T_INT", @@ -5084,7 +5090,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 }, { "item_type":"T_INT", @@ -5122,7 +5129,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 }, { "item_type":"T_INT", @@ -5160,7 +5168,8 @@ "dblink_name":"", "dblink_id":-1, "local_session_var": { }, - "local_session_var_id":-1 + "local_session_var_id":-1, + "mview_id":-1 } [34] X not like Y {