From 13c335d128c65a367c3576490dc0ba246266d0ff Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 9 Feb 2024 13:38:33 +0000 Subject: [PATCH] [CP] [OBKV] cherry-pick bugfix from 421 --- src/observer/ob_rpc_processor_simple.cpp | 1 - .../ob_table_batch_execute_processor.cpp | 12 +++- src/observer/table/ob_table_context.cpp | 3 +- src/observer/table/ob_table_context.h | 4 ++ .../table/ob_table_execute_processor.cpp | 2 +- src/observer/table/ob_table_op_wrapper.cpp | 2 +- .../ob_table_query_and_mutate_processor.cpp | 4 +- .../table/ob_table_query_processor.cpp | 4 +- .../table/ob_table_query_sync_processor.cpp | 9 ++- .../table/ob_table_query_sync_processor.h | 1 + src/observer/table/ob_table_rpc_processor.cpp | 26 +++++-- src/observer/table/ob_table_rpc_processor.h | 1 + src/observer/table/ttl/ob_table_ttl_task.cpp | 11 +-- .../table/ttl/ob_tenant_tablet_ttl_mgr.cpp | 21 +++--- .../table/ttl/ob_tenant_tablet_ttl_mgr.h | 8 ++- .../table/ttl/ob_tenant_ttl_manager.cpp | 70 +++++++++---------- .../table/ttl/ob_tenant_ttl_manager.h | 4 +- src/share/table/ob_table.h | 4 +- src/share/table/ob_ttl_util.cpp | 65 ++++++++++++----- src/share/table/ob_ttl_util.h | 8 ++- src/sql/executor/ob_cmd_executor.cpp | 3 +- src/sql/parser/sql_parser_mysql_mode.y | 2 +- .../resolver/ddl/ob_alter_table_resolver.cpp | 1 + src/sql/resolver/ddl/ob_ddl_resolver.cpp | 34 +++++---- .../table_api/r/mysql/ttl_ddl.result | 66 ++++++++++++----- .../test_suite/table_api/t/ttl_ddl.test | 61 ++++++++++++---- 26 files changed, 287 insertions(+), 140 deletions(-) diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index ae1ec196d8..d3a78a56b8 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -2973,7 +2973,6 @@ int ObTenantTTLP::process() RS_LOG(WARN, "fail to launch ttl", KR(ret), K(req)); } res.err_code_ = ret; - ret = OB_SUCCESS; return ret; } diff --git a/src/observer/table/ob_table_batch_execute_processor.cpp b/src/observer/table/ob_table_batch_execute_processor.cpp index 01c10f237c..8cbcb972ed 100644 --- a/src/observer/table/ob_table_batch_execute_processor.cpp +++ b/src/observer/table/ob_table_batch_execute_processor.cpp @@ -99,10 +99,14 @@ void ObTableBatchExecuteP::audit_on_finish() { audit_record_.consistency_level_ = ObTableConsistencyLevel::STRONG == arg_.consistency_level_ ? ObConsistencyLevel::STRONG : ObConsistencyLevel::WEAK; - audit_record_.return_rows_ = arg_.returning_affected_rows_ ? result_.count() : 0; audit_record_.table_scan_ = false; - audit_record_.affected_rows_ = result_.count(); audit_record_.try_cnt_ = retry_count_ + 1; + audit_record_.return_rows_ = 0; + audit_record_.affected_rows_ = 0; + for (int i = 0; i < result_.count(); i++) { + audit_record_.return_rows_ += result_.at(i).get_return_rows(); + audit_record_.affected_rows_ += result_.at(i).get_affected_rows(); + } } uint64_t ObTableBatchExecuteP::get_request_checksum() @@ -443,7 +447,9 @@ int ObTableBatchExecuteP::multi_get() op_result.set_entity(*result_entity); op_result.set_err(ret); op_result.set_type(tb_ctx_.get_opertion_type()); - if (OB_FAIL(result_.push_back(op_result))) { + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(result_.push_back(op_result))) { LOG_WARN("fail to push back op result", K(ret), K(i)); } else if (batch_ops_atomic_ && OB_FAIL(op_result.get_errno())) { LOG_WARN("fail to execute one operation when batch execute as atomic", K(ret), K(table_operation)); diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index 47ccf96a3d..c415b2b35b 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -782,6 +782,7 @@ int ObTableCtx::generate_key_range(const ObIArray &scan_ranges) // check obj type in ranges for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { // foreach range const ObNewRange &range = scan_ranges.at(i); + is_full_table_scan_ = is_full_table_scan_ ? is_full_table_scan_ : range.is_whole_range(); // check column type for (int64_t j = 0; OB_SUCCESS == ret && j < 2; ++j) { const ObRowkey *p_key = nullptr; @@ -1621,7 +1622,7 @@ int ObTableCtx::init_dml_related_tid() found = true; } } - if (found && OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { + if (OB_SUCC(ret) && found && OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { LOG_WARN("fail to add related index ids", K(ret), K(index_schema->get_table_id())); } } else if (OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { diff --git a/src/observer/table/ob_table_context.h b/src/observer/table/ob_table_context.h index c89e9e5eec..4836b2ef6e 100644 --- a/src/observer/table/ob_table_context.h +++ b/src/observer/table/ob_table_context.h @@ -184,6 +184,7 @@ public: is_skip_scan_ = false; is_client_set_put_ = false; binlog_row_image_type_ = ObBinlogRowImage::FULL; + is_full_table_scan_ = false; } virtual ~ObTableCtx() {} @@ -276,6 +277,7 @@ public: OB_INLINE const common::ObIArray& get_query_col_ids() const { return query_col_ids_; } OB_INLINE const common::ObIArray& get_query_col_names() const { return query_col_names_; } OB_INLINE bool is_total_quantity_log() const { return binlog_row_image_type_ == ObBinlogRowImage::FULL; } + OB_INLINE bool is_full_table_scan() const { return is_full_table_scan_; } // for update OB_INLINE bool is_for_update() const { return is_for_update_; } OB_INLINE bool is_inc_or_append() const @@ -522,6 +524,8 @@ private: // for put bool is_client_set_put_; int64_t binlog_row_image_type_; + // for audit + bool is_full_table_scan_; private: DISALLOW_COPY_AND_ASSIGN(ObTableCtx); }; diff --git a/src/observer/table/ob_table_execute_processor.cpp b/src/observer/table/ob_table_execute_processor.cpp index 304ef5f774..fae0c502a7 100644 --- a/src/observer/table/ob_table_execute_processor.cpp +++ b/src/observer/table/ob_table_execute_processor.cpp @@ -292,7 +292,7 @@ void ObTableApiExecuteP::audit_on_finish() { audit_record_.consistency_level_ = ObTableConsistencyLevel::STRONG == arg_.consistency_level_ ? ObConsistencyLevel::STRONG : ObConsistencyLevel::WEAK; - audit_record_.return_rows_ = arg_.returning_affected_rows_ ? 1 : 0; + audit_record_.return_rows_ = result_.get_return_rows(); audit_record_.table_scan_ = false; audit_record_.affected_rows_ = result_.get_affected_rows(); audit_record_.try_cnt_ = retry_count_ + 1; diff --git a/src/observer/table/ob_table_op_wrapper.cpp b/src/observer/table/ob_table_op_wrapper.cpp index aa7c3c7b1c..68aa70bc68 100644 --- a/src/observer/table/ob_table_op_wrapper.cpp +++ b/src/observer/table/ob_table_op_wrapper.cpp @@ -413,7 +413,7 @@ int ObHTableDeleteExecutor::build_range(ObTableQuery &query) range.border_flag_.set_inclusive_start(); range.border_flag_.set_inclusive_end(); - if (OB_FAIL(key_ranges.push_back(range))) { + if (OB_SUCC(ret) && OB_FAIL(key_ranges.push_back(range))) { LOG_WARN("fail to push back hdelete scan range", K(ret), K(key_ranges)); } diff --git a/src/observer/table/ob_table_query_and_mutate_processor.cpp b/src/observer/table/ob_table_query_and_mutate_processor.cpp index 94fa0e8f0f..9c2eabc96c 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.cpp +++ b/src/observer/table/ob_table_query_and_mutate_processor.cpp @@ -98,8 +98,8 @@ int ObTableQueryAndMutateP::check_arg() void ObTableQueryAndMutateP::audit_on_finish() { audit_record_.consistency_level_ = ObConsistencyLevel::STRONG; // todo: exact consistency - audit_record_.return_rows_ = arg_.query_and_mutate_.return_affected_entity() ? result_.affected_entity_.get_row_count() : 0; - audit_record_.table_scan_ = true; // todo: exact judgement + audit_record_.return_rows_ = result_.affected_entity_.get_row_count(); + audit_record_.table_scan_ = tb_ctx_.is_full_table_scan(); audit_record_.affected_rows_ = result_.affected_rows_; audit_record_.try_cnt_ = retry_count_ + 1; } diff --git a/src/observer/table/ob_table_query_processor.cpp b/src/observer/table/ob_table_query_processor.cpp index 295420fbab..7556fd1025 100644 --- a/src/observer/table/ob_table_query_processor.cpp +++ b/src/observer/table/ob_table_query_processor.cpp @@ -67,8 +67,8 @@ void ObTableQueryP::audit_on_finish() audit_record_.consistency_level_ = ObTableConsistencyLevel::STRONG == arg_.consistency_level_ ? ObConsistencyLevel::STRONG : ObConsistencyLevel::WEAK; audit_record_.return_rows_ = result_.get_row_count(); - audit_record_.table_scan_ = true; // todo: exact judgement - audit_record_.affected_rows_ = result_.get_row_count(); + audit_record_.table_scan_ = tb_ctx_.is_full_table_scan(); + audit_record_.affected_rows_ = 0; audit_record_.try_cnt_ = retry_count_ + 1; } diff --git a/src/observer/table/ob_table_query_sync_processor.cpp b/src/observer/table/ob_table_query_sync_processor.cpp index 8a9588ae75..6f8763b67c 100644 --- a/src/observer/table/ob_table_query_sync_processor.cpp +++ b/src/observer/table/ob_table_query_sync_processor.cpp @@ -277,7 +277,8 @@ ObTableQuerySyncP::ObTableQuerySyncP(const ObGlobalContext &gctx) result_row_count_(0), query_session_id_(0), allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), - query_session_(nullptr) + query_session_(nullptr), + is_full_table_scan_(false) {} int ObTableQuerySyncP::deserialize() @@ -308,8 +309,8 @@ void ObTableQuerySyncP::audit_on_finish() ? ObConsistencyLevel::STRONG : ObConsistencyLevel::WEAK; audit_record_.return_rows_ = result_.get_row_count(); - audit_record_.table_scan_ = true; // todo: exact judgement - audit_record_.affected_rows_ = result_.get_row_count(); + audit_record_.table_scan_ = is_full_table_scan_; + audit_record_.affected_rows_ = 0; audit_record_.try_cnt_ = retry_count_ + 1; } @@ -528,6 +529,7 @@ int ObTableQuerySyncP::query_scan_with_init() } else { audit_row_count_ = result_.get_row_count(); result_.query_session_id_ = query_session_id_; + is_full_table_scan_ = tb_ctx.is_full_table_scan(); } return ret; @@ -562,6 +564,7 @@ int ObTableQuerySyncP::query_scan_without_init() result_.is_end_ = !result_iter->has_more_result(); result_.query_session_id_ = query_session_id_; audit_row_count_ = result_.get_row_count(); + is_full_table_scan_ = tb_ctx.is_full_table_scan(); } } diff --git a/src/observer/table/ob_table_query_sync_processor.h b/src/observer/table/ob_table_query_sync_processor.h index a6f6612ca6..aaca098c96 100644 --- a/src/observer/table/ob_table_query_sync_processor.h +++ b/src/observer/table/ob_table_query_sync_processor.h @@ -219,6 +219,7 @@ private: ObArenaAllocator allocator_; ObTableQuerySyncSession *query_session_; int64_t timeout_ts_; + bool is_full_table_scan_; }; } // end namespace observer diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index e4c0dd8633..5fceb89493 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -32,6 +32,8 @@ #include "ob_table_move_response.h" #include "ob_table_connection_mgr.h" #include "share/table/ob_table_util.h" +#include "observer/mysql/obmp_base.h" +#include "lib/stat/ob_session_stat.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -248,7 +250,8 @@ ObTableApiProcessorBase::ObTableApiProcessorBase(const ObGlobalContext &gctx) retry_count_(0), trans_desc_(NULL), had_do_response_(false), - user_client_addr_() + user_client_addr_(), + sess_stat_guard_(MTL_ID(), ObActiveSessionGuard::get_stat().session_id_) { need_audit_ = GCONF.enable_sql_audit; trans_state_ptr_ = &trans_state_; @@ -625,7 +628,9 @@ void ObTableApiProcessorBase::start_audit(const rpc::ObRequest *req) { if (OB_LIKELY(NULL != req)) { - audit_record_.user_client_addr_ = RPC_REQ_OP.get_peer(req); + audit_record_.user_client_addr_ = ObCurTraceId::get_addr(); + audit_record_.client_addr_ = RPC_REQ_OP.get_peer(req); + audit_record_.trace_id_ = req->get_trace_id(); save_request_string(); @@ -658,8 +663,6 @@ static int set_audit_name(const char *info_name, char *&audit_name, int64_t &aud void ObTableApiProcessorBase::end_audit() { // credential info -// audit_record_.server_addr_; // not necessary, because gv_sql_audit_iterator use local addr automatically -// audit_record_.client_addr_; // not used for now audit_record_.tenant_id_ = credential_.tenant_id_; audit_record_.effective_tenant_id_ = credential_.tenant_id_; audit_record_.user_id_ = credential_.user_id_; @@ -685,6 +688,7 @@ void ObTableApiProcessorBase::end_audit() } { // set user name, ignore ret + ret = OB_SUCCESS; const share::schema::ObUserInfo *user_info = NULL; if(OB_FAIL(schema_guard.get_user_info(credential_.tenant_id_, credential_.user_id_, user_info))) { SERVER_LOG(WARN, "fail to get user info", K(ret), K(credential_)); @@ -698,6 +702,7 @@ void ObTableApiProcessorBase::end_audit() } { // set database name, ignore ret + ret = OB_SUCCESS; const share::schema::ObSimpleDatabaseSchema *database_info = NULL; if(OB_FAIL(schema_guard.get_database_schema(credential_.tenant_id_, credential_.database_id_, database_info))) { SERVER_LOG(WARN, "fail to get database info", K(ret), K(credential_)); @@ -746,11 +751,19 @@ void ObTableApiProcessorBase::end_audit() audit_record_.seq_ = 0; // not used audit_record_.session_id_ = 0; // not used for table api + // tx info + audit_record_.trans_id_ = tx_snapshot_.core_.tx_id_.get_id(); + audit_record_.snapshot_.version_ = tx_snapshot_.core_.version_; + audit_record_.snapshot_.tx_id_ = tx_snapshot_.core_.tx_id_.get_id(); + audit_record_.snapshot_.scn_ = tx_snapshot_.core_.scn_.cast_to_int(); + audit_record_.snapshot_.source_ = tx_snapshot_.get_source_name(); + const int64_t elapsed_time = common::ObTimeUtility::current_time() - audit_record_.exec_timestamp_.receive_ts_; if (elapsed_time > GCONF.trace_log_slow_query_watermark) { FORCE_PRINT_TRACE(THE_TRACE, "[table api][slow query]"); } + ret = OB_SUCCESS; MTL_SWITCH(credential_.tenant_id_) { obmysql::ObMySQLRequestManager *req_manager = MTL(obmysql::ObMySQLRequestManager*); if (nullptr == req_manager) { @@ -786,6 +799,8 @@ int ObTableApiProcessorBase::process_with_retry(const ObString &credential, cons ObMaxWaitGuard max_wait_guard(&audit_record_.exec_record_.max_wait_event_); ObTotalWaitGuard total_wait_guard(&total_wait_desc); ObTenantStatEstGuard stat_guard(credential_.tenant_id_); + ObProcessMallocCallback pmcb(0, audit_record_.request_memory_used_); + ObMallocCallbackGuard malloc_guard_(pmcb); need_retry_in_queue_ = false; bool did_local_retry = false; do { @@ -977,8 +992,7 @@ int ObTableRpcProcessor::after_process(int error_code) NG_TRACE(process_end); // print trace log if necessary // some statistics must be recorded for plan stat, even though sql audit disabled audit_record_.exec_timestamp_.exec_type_ = ExecType::RpcProcessor; - audit_record_.exec_timestamp_.net_t_ = - audit_record_.exec_timestamp_.receive_ts_ - audit_record_.exec_timestamp_.rpc_send_ts_; + audit_record_.exec_timestamp_.net_t_ = 0; audit_record_.exec_timestamp_.net_wait_t_ = audit_record_.exec_timestamp_.enter_queue_ts_ - audit_record_.exec_timestamp_.receive_ts_; audit_record_.exec_timestamp_.update_stage_time(); diff --git a/src/observer/table/ob_table_rpc_processor.h b/src/observer/table/ob_table_rpc_processor.h index 17731e9bd5..bd5951ee34 100644 --- a/src/observer/table/ob_table_rpc_processor.h +++ b/src/observer/table/ob_table_rpc_processor.h @@ -192,6 +192,7 @@ protected: sql::TransState *trans_state_ptr_; transaction::ObTxReadSnapshot tx_snapshot_; ObAddr user_client_addr_; + ObSessionStatEstGuard sess_stat_guard_; }; template diff --git a/src/observer/table/ttl/ob_table_ttl_task.cpp b/src/observer/table/ttl/ob_table_ttl_task.cpp index 6bff0ab988..0e6cf65d0e 100644 --- a/src/observer/table/ttl/ob_table_ttl_task.cpp +++ b/src/observer/table/ttl/ob_table_ttl_task.cpp @@ -524,7 +524,11 @@ int ObTableTTLDeleteRowIterator::get_next_row(ObNewRow*& row) } else { bool is_expired = false; while(OB_SUCC(ret) && !is_expired) { - if (OB_FAIL(ObTableApiScanRowIterator::get_next_row(row))) { + int64_t cur_ts = ObTimeUtility::current_time(); + if (cur_ts > iter_end_ts_ && hbase_new_cq_) { + ret = OB_ITER_END; + LOG_DEBUG("iter_end_ts reached, stop current iterator", KR(ret), K(cur_ts), K_(iter_end_ts)); + } else if (OB_FAIL(ObTableApiScanRowIterator::get_next_row(row))) { if (OB_ITER_END != ret) { LOG_WARN("fail to get next row", K(ret)); } @@ -552,6 +556,8 @@ int ObTableTTLDeleteRowIterator::get_next_row(ObNewRow*& row) } else { cur_version_++; } + // NOTE: after ttl_cnt_ or cur_del_rows_ is incremented, the row must be return to delete iterator + // cuz we will check the affected_rows correctness after finish delete. if (max_version_ > 0 && cur_version_ > max_version_) { max_version_cnt_++; cur_del_rows_++; @@ -576,9 +582,6 @@ int ObTableTTLDeleteRowIterator::get_next_row(ObNewRow*& row) } } } - if (ObTimeUtility::current_time() > iter_end_ts_ && hbase_new_cq_) { - ret = OB_ITER_END; - } } } diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp index 7350a79832..0755752647 100644 --- a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp @@ -315,6 +315,7 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask // lock task ctx for update common::ObSpinLockGuard ctx_guard(ctx->lock_); ctx->last_modify_time_ = ObTimeUtility::current_time(); + ctx->in_queue_ = false; mark_ttl_ctx_dirty(local_tenant_task_, *ctx); if (need_copy_task && OB_FAIL(deep_copy_task(ctx, task_info, task_para))) { LOG_WARN("fail to deep copy task", KR(ret), K_(tenant_id), K(task_info), K(task_para)); @@ -904,8 +905,10 @@ int ObTenantTabletTTLMgr::sync_sys_table_op(ObTTLTaskCtx* ctx, } } - // check tenant ttl status - if (OB_SUCC(ret) && OB_FAIL(ObTTLUtil::check_tenant_state(tenant_id_, trans, local_tenant_task_.state_, local_tenant_task_.task_id_, tenant_state_changed))) { + // check and ensure the tenant status not change in this transaction + // when tablet task is adready in terminal status, do not need check tenant state + // beause we won't change tablet status in such case + if (OB_SUCC(ret) && !is_end_state && OB_FAIL(ObTTLUtil::check_tenant_state(tenant_id_, trans, local_tenant_task_.state_, local_tenant_task_.task_id_, tenant_state_changed))) { FLOG_INFO("local tenant task state is different from sys table", KR(ret), K_(tenant_id), K(local_tenant_task_.state_)); } @@ -978,7 +981,7 @@ int ObTenantTabletTTLMgr::sync_sys_table(ObTabletID& tablet_id, bool &tenant_sta } } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && !ctx->in_queue_) { switch (ctx->task_status_) { case OB_TTL_TASK_PREPARE: { if (OB_FAIL(sync_sys_table_op(ctx, false, tenant_state_changed))) { @@ -1007,15 +1010,10 @@ int ObTenantTabletTTLMgr::sync_sys_table(ObTabletID& tablet_id, bool &tenant_sta } } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && !ctx->in_queue_) { //mark ctx dirty false - if (OB_ISNULL(ctx)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ctx is null", KR(ret)); - } else { - ctx->is_dirty_ = false; - LOG_INFO("finish mark ctx dirty false", KR(ret), K(tablet_id), KPC(ctx)); - } + ctx->is_dirty_ = false; + LOG_INFO("finish mark ctx dirty false", KR(ret), K(tablet_id), KPC(ctx)); } return ret; } @@ -1191,6 +1189,7 @@ int ObTenantTabletTTLMgr::try_schedule_task(ObTTLTaskCtx* ctx) if (ctx->task_start_time_ == OB_INVALID_ID) { ctx->task_start_time_ = ObTimeUtility::current_time(); } + ctx->in_queue_ = true; ctx->task_status_ = OB_TTL_TASK_RUNNING; // mark ctx dirty later in report_task_status in case of watting too long in dag queue } diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h index 92378e0804..fb2955a984 100644 --- a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h @@ -34,7 +34,8 @@ public : last_modify_time_(OB_INVALID_ID), failure_times_(0), is_dirty_(false), - need_refresh_(true) {} + need_refresh_(true), + in_queue_(false) {} bool is_valid() { return task_info_.is_valid() && ttl_para_.is_valid(); @@ -42,8 +43,8 @@ public : int deep_copy_rowkey(const ObString &rowkey); - TO_STRING_KV(K_(task_info), K_(task_status), K_(ttl_para), K_(task_start_time), - K_(last_modify_time), K_(failure_times), K_(is_dirty), K_(need_refresh)); + TO_STRING_KV(K_(task_info), K_(task_status), K_(ttl_para), K_(task_start_time), K_(last_modify_time), + K_(failure_times), K_(is_dirty), K_(need_refresh), K_(in_queue)); public: common::ObArenaAllocator rowkey_cp_allcoator_; // for rowkey copy in ObTTLTaskInfo @@ -60,6 +61,7 @@ public: bool is_moved_; bool need_refresh_; // should refresh task from task table common::ObSpinLock lock_; // lock for update + bool in_queue_; // whether in dag queue or not }; class ObTenantTabletTTLMgr; diff --git a/src/observer/table/ttl/ob_tenant_ttl_manager.cpp b/src/observer/table/ttl/ob_tenant_ttl_manager.cpp index 18c440d598..2e6c609d75 100644 --- a/src/observer/table/ttl/ob_tenant_ttl_manager.cpp +++ b/src/observer/table/ttl/ob_tenant_ttl_manager.cpp @@ -119,8 +119,6 @@ int ObTTLTaskScheduler::reload_tenant_task() if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ttl tenant task mgr not init", KR(ret)); - } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { - // do nothing } else if (ATOMIC_BCAS(&need_reload_, true, false)) { lib::ObMutexGuard guard(mutex_); SMART_VAR(ObMySQLProxy::MySQLResult, res) { @@ -191,12 +189,7 @@ int ObTTLTaskScheduler::calc_next_task_state(ObTTLTaskType user_cmd_type, ObTTLTaskStatus &next_state) { int ret = OB_SUCCESS; - if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("fail to modify ttl tasks status, the state is not mismatch, moving now", - KR(ret), K(curr_state), K(user_cmd_type)); - LOG_USER_WARN(OB_NOT_SUPPORTED, "Change the current TTL task state(Move) to the destination state"); - } else if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_SUSPEND && + if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_SUSPEND && user_cmd_type == ObTTLTaskType::OB_TTL_RESUME) { next_state = ObTTLTaskStatus::OB_RS_TTL_TASK_CREATE; } else if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_CREATE && @@ -206,10 +199,11 @@ int ObTTLTaskScheduler::calc_next_task_state(ObTTLTaskType user_cmd_type, user_cmd_type == ObTTLTaskType::OB_TTL_CANCEL) { next_state = ObTTLTaskStatus::OB_RS_TTL_TASK_CANCEL; } else { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("fail to modify ttl tasks status, the state is mismatch. Maybe previous tasks is running", - KR(ret), K(curr_state), K(user_cmd_type)); - LOG_USER_WARN(OB_NOT_SUPPORTED, "Change the current TTL task state to the destination state"); + ret = OB_TTL_CMD_NOT_ALLOWED; + const char *status_cstr = ObTTLUtil::get_ttl_tenant_status_cstr(curr_state); + LOG_USER_ERROR(OB_TTL_CMD_NOT_ALLOWED, status_cstr); + LOG_WARN("ttl command is not allowed in current tenant ttl status", + KR(ret), K(curr_state), K(user_cmd_type), K_(tenant_id)); } return ret; } @@ -222,8 +216,9 @@ int ObTTLTaskScheduler::add_ttl_task(ObTTLTaskType task_type) ret = OB_NOT_INIT; LOG_WARN("ttl tenant task mgr not init", KR(ret)); } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("cann't process ttl task, maybe tenant is restoring", K_(tenant_id), KR(ret)); + ret = OB_TTL_TENANT_IS_RESTORE; + LOG_USER_ERROR(OB_TTL_TENANT_IS_RESTORE); + LOG_WARN("cannot execute ttl task during tenant is restoring", K_(tenant_id), KR(ret)); } else if (OB_FAIL(reload_tenant_task())) { LOG_WARN("fail to reload tenant task", KR(ret)); } else { @@ -251,9 +246,10 @@ int ObTTLTaskScheduler::add_ttl_task(ObTTLTaskType task_type) } else { // not task or task finished already, only accept trigger command if (task_type != ObTTLTaskType::OB_TTL_TRIGGER) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("not ttl task currently, only trigger command is supported", KR(ret), K(task_type)); - LOG_USER_WARN(OB_NOT_SUPPORTED, "trigger TTL task when TTL task is executing"); + ret = OB_TTL_NO_TASK_RUNNING; + LOG_USER_ERROR(OB_TTL_NO_TASK_RUNNING); + LOG_WARN("not ttl task running currently, only trigger command is supported", + KR(ret), K(task_type), K(curr_state)); } else if (OB_FAIL(add_ttl_task_internal(TRIGGER_TYPE::USER_TRIGGER))) { LOG_WARN("fail to add ttl task", KR(ret), K_(tenant_id)); } @@ -268,13 +264,14 @@ int ObTTLTaskScheduler::add_ttl_task_internal(TRIGGER_TYPE trigger_type) { int ret = OB_SUCCESS; bool is_active_time = false; - bool enable_ttl = is_enable_ttl(); + bool enable_ttl = ObTTLUtil::is_enable_ttl(tenant_id_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ttl tenant task mgr not init", KR(ret)); - } else if (!is_enable_ttl()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("ttl is not enable currently", KR(ret)); + } else if (!enable_ttl) { + ret = OB_TTL_NOT_ENABLE; + LOG_USER_ERROR(OB_TTL_NOT_ENABLE); + LOG_WARN("ttl is not enable currently", KR(ret), K_(tenant_id)); } else if (!tenant_task_.is_finished_) { ret = OB_STATE_NOT_MATCH; LOG_WARN("there is ttl task executing already", KR(ret)); @@ -387,17 +384,12 @@ int ObTTLTaskScheduler::in_active_time(bool& is_active_time) return ret; } -bool ObTTLTaskScheduler::is_enable_ttl() -{ - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); - return tenant_config.is_valid() && tenant_config->enable_kv_ttl; -} - int ObTTLTaskScheduler::try_add_periodic_task() { int ret = OB_SUCCESS; TRIGGER_TYPE trigger_type = TRIGGER_TYPE::PERIODIC_TRIGGER; bool is_active_time = false; + bool enable_ttl = ObTTLUtil::is_enable_ttl(tenant_id_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ttl tenant task mgr not init", KR(ret)); @@ -407,6 +399,8 @@ int ObTTLTaskScheduler::try_add_periodic_task() } else if (need_skip_run()) { ret = OB_EAGAIN; FLOG_INFO("exit timer task once cuz leader switch", KR(ret), K_(is_leader), K_(need_do_for_switch)); + } else if (!enable_ttl) { + // do nothing } else if (OB_FAIL(in_active_time(is_active_time))) { LOG_WARN("fail to check is in active time", KR(ret)); } else if (is_active_time) { @@ -428,16 +422,13 @@ int ObTTLTaskScheduler::check_all_tablet_task() { int ret = OB_SUCCESS; bool need_move = true; - bool is_cancel_task = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ttl tenant task mgr not init", KR(ret)); - } else if (tenant_task_.ttl_status_.status_ == OB_TTL_TASK_INVALID) { + } else if (tenant_task_.ttl_status_.status_ == OB_TTL_TASK_INVALID || tenant_task_.ttl_status_.status_ == OB_RS_TTL_TASK_SUSPEND) { // do nothing } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { // do nothing - } else if (FALSE_IT(is_cancel_task = (tenant_task_.ttl_status_.status_ == ObTTLTaskStatus::OB_RS_TTL_TASK_CANCEL)? true : false)) { - // do nothing } else if (OB_FAIL(check_task_need_move(need_move))) { LOG_WARN("fail to check task need move", KR(ret), K_(tenant_id)); } else if (need_move) { @@ -446,6 +437,9 @@ int ObTTLTaskScheduler::check_all_tablet_task() if (need_skip_run()) { ret = OB_EAGAIN; FLOG_INFO("exit timer task once cuz leader switch", KR(ret), K_(is_leader), K_(need_do_for_switch)); + } else if (ObTTLTaskStatus::OB_RS_TTL_TASK_SUSPEND == tenant_task_.ttl_status_.status_) { + ret = OB_EAGAIN; + LOG_WARN("task status changed during check task need move", KR(ret)); } else if (ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE != tenant_task_.ttl_status_.status_) { tenant_task_.ttl_status_.status_ = static_cast(ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE); ObMySQLTransaction trans; @@ -465,7 +459,7 @@ int ObTTLTaskScheduler::check_all_tablet_task() } if (OB_SUCC(ret)) { - if (OB_FAIL(move_all_task_to_history_table(is_cancel_task))) { + if (OB_FAIL(move_all_task_to_history_table())) { LOG_WARN("fail to move all tasks to history table", KR(ret), K_(tenant_id), K(tenant_task_.ttl_status_.table_id_)); } else { tenant_task_.reset(); @@ -629,6 +623,8 @@ void ObTTLTaskScheduler::runTimerTask() } else if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ttl task mgr not init", KR(ret)); + } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + // do nothing } else if (ATOMIC_BCAS(&need_do_for_switch_, true, false)) { // need skip this round for waiting follower finish executing task if (is_leader_) { @@ -651,11 +647,16 @@ int ObTenantTTLManager::handle_user_ttl(const obrpc::ObTTLRequestArg& arg) { int ret = OB_SUCCESS; ObTTLTaskType user_ttl_req_type = static_cast(arg.cmd_code_); + bool enable_ttl = ObTTLUtil::is_enable_ttl(tenant_id_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); } else if (tenant_id_ == OB_SYS_TENANT_ID) { // do nothing + } else if (!enable_ttl) { + ret = OB_TTL_NOT_ENABLE; + LOG_USER_ERROR(OB_TTL_NOT_ENABLE); + LOG_WARN("ttl is not enable currently", KR(ret), K_(tenant_id)); } else if (OB_FAIL(task_scheduler_.add_ttl_task(static_cast(arg.cmd_code_)))) { LOG_WARN("fail to add ttl task", KR(ret), K_(tenant_id), K(user_ttl_req_type)); } @@ -664,7 +665,7 @@ int ObTenantTTLManager::handle_user_ttl(const obrpc::ObTTLRequestArg& arg) return ret; } -int ObTTLTaskScheduler::move_all_task_to_history_table(bool need_cancel) +int ObTTLTaskScheduler::move_all_task_to_history_table() { int ret = OB_SUCCESS; int64_t one_move_rows = TBALET_CHECK_BATCH_SIZE; @@ -676,8 +677,7 @@ int ObTTLTaskScheduler::move_all_task_to_history_table(bool need_cancel) } else if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) { LOG_WARN("fail start transaction", KR(ret), K_(tenant_id)); } else if (OB_FAIL(ObTTLUtil::move_task_to_history_table(tenant_id_, tenant_task_.ttl_status_.task_id_, - trans, TBALET_CHECK_BATCH_SIZE, one_move_rows, - need_cancel))) { + trans, TBALET_CHECK_BATCH_SIZE, one_move_rows))) { LOG_WARN("fail to move task to history table", KR(ret), K_(tenant_id)); } diff --git a/src/observer/table/ttl/ob_tenant_ttl_manager.h b/src/observer/table/ttl/ob_tenant_ttl_manager.h index 77c0691bf6..e6391cf1e6 100644 --- a/src/observer/table/ttl/ob_tenant_ttl_manager.h +++ b/src/observer/table/ttl/ob_tenant_ttl_manager.h @@ -116,8 +116,6 @@ public: void pause(); void resume(); private: - virtual bool is_enable_ttl(); - virtual int delete_task(const uint64_t tenant_id, const uint64_t task_id); virtual int in_active_time(bool& is_active_time); @@ -148,7 +146,7 @@ private: private: int check_all_tablet_finished(bool &all_finished); int check_tablet_table_finished(common::ObIArray &pairs, bool &all_finished); - int move_all_task_to_history_table(bool need_cancel); + int move_all_task_to_history_table(); OB_INLINE bool need_skip_run() { return ATOMIC_LOAD(&need_do_for_switch_); } private: static const int64_t TBALE_CHECK_BATCH_SIZE = 200; diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index 27091ee732..0115d46447 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -423,6 +423,7 @@ public: int get_entity(ObITableEntity *&entity); ObITableEntity *get_entity() { return entity_; } int64_t get_affected_rows() const { return affected_rows_; } + int get_return_rows() { return ((entity_ == NULL || entity_->is_empty()) ? 0 : 1); } void set_entity(ObITableEntity &entity) { entity_ = &entity; } void set_type(ObTableOperationType::Type op_type) { operation_type_ = op_type; } @@ -842,7 +843,8 @@ public: uint64_t get_checksum(); TO_STRING_KV(K_(query), - K_(mutations)); + K_(mutations), + K_(return_affected_entity)); private: ObTableQuery query_; ObTableBatchOperation mutations_; diff --git a/src/share/table/ob_ttl_util.cpp b/src/share/table/ob_ttl_util.cpp index 7ad6d1ad89..f3f1e0bc28 100644 --- a/src/share/table/ob_ttl_util.cpp +++ b/src/share/table/ob_ttl_util.cpp @@ -592,23 +592,13 @@ bool ObTTLUtil::check_can_process_tenant_tasks(uint64_t tenant_id) int ObTTLUtil::move_task_to_history_table(uint64_t tenant_id, uint64_t task_id, common::ObMySQLTransaction& proxy, - int64_t batch_size, int64_t &move_rows, - bool need_cancel) + int64_t batch_size, int64_t &move_rows) { int ret = OB_SUCCESS; ObSqlString sql; int64_t insert_rows = 0; int64_t delete_rows = 0; - if (!need_cancel && - OB_FAIL(sql.assign_fmt("replace into %s select * from %s " - " where task_id = %ld and tablet_id != -1 and table_id != -1" - " order by tenant_id, task_id, table_id, tablet_id LIMIT %ld", - share::OB_ALL_KV_TTL_TASK_HISTORY_TNAME, - share::OB_ALL_KV_TTL_TASK_TNAME, - task_id, batch_size))) { - LOG_WARN("sql assign fmt failed", K(ret)); - } else if (need_cancel && - OB_FAIL(sql.assign_fmt("replace into %s select gmt_create, gmt_modified," + if (OB_FAIL(sql.assign_fmt("replace into %s select gmt_create, gmt_modified," " tenant_id, task_id, table_id, tablet_id, task_start_time," " task_update_time, trigger_type, if(status=4, 4, 3) as status," " ttl_del_cnt, max_version_del_cnt, scan_cnt, row_key, ret_code from %s" @@ -731,9 +721,9 @@ int ObTTLUtil::parse_kv_attributes(const ObString &kv_attributes, int32_t &max_v json::Value *ttl_val = elem->value_; if (NULL != ttl_val && ttl_val->get_type() == json::JT_NUMBER) { if (ttl_val->get_number() <= 0) { - ret = OB_INVALID_ARGUMENT; + ret = OB_TTL_INVALID_HBASE_TTL; LOG_WARN("time to live should greater than 0", K(ret), K(ttl_val)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "time to live, which should greater than 0"); + LOG_USER_ERROR(OB_TTL_INVALID_HBASE_TTL); } else { time_to_live = static_cast(ttl_val->get_number()); } @@ -742,9 +732,9 @@ int ObTTLUtil::parse_kv_attributes(const ObString &kv_attributes, int32_t &max_v json::Value *max_versions_val = elem->value_; if (NULL != max_versions_val && max_versions_val->get_type() == json::JT_NUMBER) { if (max_versions_val->get_number() <= 0) { - ret = OB_INVALID_ARGUMENT; + ret = OB_TTL_INVALID_HBASE_MAXVERSIONS; LOG_WARN("max versions should greater than 0", K(ret), K(max_versions_val)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "max versions, which should greater than 0"); + LOG_USER_ERROR(OB_TTL_INVALID_HBASE_MAXVERSIONS); } else { max_versions = static_cast(max_versions_val->get_number()); } @@ -771,7 +761,7 @@ int ObTTLUtil::parse_kv_attributes(const ObString &kv_attributes, int32_t &max_v } } } else { - ret = OB_INVALID_ARGUMENT; + ret = OB_NOT_SUPPORTED; LOG_WARN("not supported kv attribute", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes with wrong format"); } @@ -1205,5 +1195,46 @@ int ObTTLUtil::check_task_status_from_sys_table(uint64_t tenant_id, common::ObIS } +bool ObTTLUtil::is_enable_ttl(uint64_t tenant_id) +{ + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + return tenant_config.is_valid() && tenant_config->enable_kv_ttl; +} + +const char * ObTTLUtil::get_ttl_tenant_status_cstr(const ObTTLTaskStatus &status) +{ + const char *status_cstr = NULL; + + switch (status) { + case OB_RS_TTL_TASK_CREATE: { + status_cstr = "RUNNING"; + break; + } + case OB_RS_TTL_TASK_SUSPEND: { + status_cstr = "PENDING"; + break; + } + case OB_RS_TTL_TASK_CANCEL: { + status_cstr = "CANCELING"; + break; + } + case OB_RS_TTL_TASK_MOVE: { + status_cstr = "MOVING"; + break; + } + case OB_TTL_TASK_FINISH: { // wait + status_cstr = "FINISHED"; + break; + } + default: { + status_cstr = "UNKNOWN"; + break; + } + } + + return status_cstr; +} + + } // end namespace rootserver } // end namespace oceanbase \ No newline at end of file diff --git a/src/share/table/ob_ttl_util.h b/src/share/table/ob_ttl_util.h index 43e7ca95e8..f70fb27f41 100644 --- a/src/share/table/ob_ttl_util.h +++ b/src/share/table/ob_ttl_util.h @@ -263,7 +263,8 @@ public: obrpc::ObTTLRequestArg::TTLRequestType type_; }; -class ObTTLUtil { +class ObTTLUtil +{ public: static int parse(const char* str, ObTTLDutyDuration& duration); static bool current_in_duration(ObTTLDutyDuration& duration); @@ -317,8 +318,7 @@ public: static int move_task_to_history_table(uint64_t tenant_id, uint64_t task_id, common::ObMySQLTransaction& proxy, - int64_t batch_size, int64_t &move_rows, - bool need_cancel = false); + int64_t batch_size, int64_t &move_rows); static int move_tenant_task_to_history_table(uint64_t tenant_id, uint64_t task_id, common::ObMySQLTransaction& proxy); @@ -340,6 +340,8 @@ public: static inline bool is_ttl_task_status_end_state(ObTTLTaskStatus status) { return status == ObTTLTaskStatus::OB_TTL_TASK_CANCEL || status == ObTTLTaskStatus::OB_TTL_TASK_FINISH; } + static bool is_enable_ttl(uint64_t tenant_id); + static const char *get_ttl_tenant_status_cstr(const ObTTLTaskStatus &status); const static uint64_t TTL_TENNAT_TASK_TABLET_ID = -1; const static uint64_t TTL_TENNAT_TASK_TABLE_ID = -1; private: diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 68bb009ca8..74f964870d 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -1062,7 +1062,8 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) "cmd_type", cmd.get_cmd_type(), "sql_text", ObHexEscapeSqlStr(ctx.get_sql_ctx()->is_sensitive_ ? ObString(OB_MASKED_STR) : sql_text), - "return_code", ret); + "return_code", ret, + "tenant_id", MTL_ID()); } if (is_ddl_or_dcl_stmt) { diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index e65ac19502..9217d3d863 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -6763,7 +6763,7 @@ ttl_expr ; ttl_expr: -simple_expr '+' INTERVAL INTNUM ttl_unit +column_definition_ref '+' INTERVAL INTNUM ttl_unit { malloc_non_terminal_node($$, result->malloc_pool_, T_TTL_EXPR, 3, $1, $4, $5); dup_expr_string($$, result, @1.first_column, @1.last_column); diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 6b4edebc62..960b46f70a 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -1109,6 +1109,7 @@ int ObAlterTableResolver::resolve_action_list(const ParseNode &node) } else if (is_ttl_column(column->get_origin_column_name(), ttl_columns)) { ret = OB_NOT_SUPPORTED; LOG_WARN("Modify/Change TTL column is not allowed", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Modify/Change TTL column"); } } } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 8bb0f2ff4f..695d908212 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -11394,20 +11394,28 @@ int ObDDLResolver::check_ttl_definition(const ParseNode *node) LOG_WARN("not supported statement for TTL expression", K(ret), K(stmt_->get_stmt_type())); } - if (OB_SUCC(ret)) { - ObString ttl_definition(node->str_len_, node->str_value_); - ObSEArray ttl_columns; - if (OB_FAIL(get_ttl_columns(ttl_definition, ttl_columns))) { - LOG_WARN("fail to get ttl columns", K(ttl_definition)); - } - for (int i = 0; i < OB_SUCC(ret) && ttl_columns.count(); i++) { - ObString column_name = ttl_columns.at(i); - if (NULL == (column_schema = tbl_schema->get_column_schema(column_name))) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("ttl column is invalid", K(ret)); + for (int i = 0; OB_SUCC(ret) && i < node->num_child_; ++i) { + if (OB_ISNULL(node->children_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ttl expr is null", K(ret), K(i)); + } else if (OB_ISNULL(node->children_[i]) || T_TTL_EXPR != node->children_[i]->type_ || + node->children_[i]->num_child_ != 3) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child node of ttl definition is wrong", KR(ret), K(node->children_[i])); + } else if (OB_ISNULL(node->children_[i]->children_[0]) || T_COLUMN_REF != node->children_[i]->children_[0]->type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child node of ttl expr is wrong", KR(ret), K(node->children_[i]->children_[0])); + } else { + ObString column_name(node->children_[i]->children_[0]->str_len_, node->children_[i]->children_[0]->str_value_); + if (OB_ISNULL(column_schema = tbl_schema->get_column_schema(column_name))) { + ret = OB_TTL_COLUMN_NOT_EXIST; + LOG_USER_ERROR(OB_TTL_COLUMN_NOT_EXIST, column_name.length(), column_name.ptr()); + LOG_WARN("ttl column is not exists", K(ret), K(column_name)); } else if ((!ob_is_datetime_tc(column_schema->get_data_type()))) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("invalid ttl expression, ttl column type should be datetime or timestamp", K(ret), K(column_schema->get_data_type())); + ret = OB_TTL_COLUMN_TYPE_NOT_SUPPORTED; + LOG_USER_ERROR(OB_TTL_COLUMN_TYPE_NOT_SUPPORTED, column_name.length(), column_name.ptr()); + LOG_WARN("invalid ttl expression, ttl column type should be datetime or timestamp", + K(ret), K(column_name), K(column_schema->get_data_type())); } } } diff --git a/tools/deploy/mysql_test/test_suite/table_api/r/mysql/ttl_ddl.result b/tools/deploy/mysql_test/test_suite/table_api/r/mysql/ttl_ddl.result index 1edbf3e645..751526ace3 100644 --- a/tools/deploy/mysql_test/test_suite/table_api/r/mysql/ttl_ddl.result +++ b/tools/deploy/mysql_test/test_suite/table_api/r/mysql/ttl_ddl.result @@ -14,7 +14,7 @@ T bigint NOT NULL, V varbinary(1024) DEFAULT NULL, PRIMARY KEY (K, Q, T) ) kv_attributes ='{}' partition by key(K) partitions 3; -ERROR HY000: Invalid argument +ERROR 0A000: kv attributes with wrong format not supported CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -38,7 +38,7 @@ T bigint NOT NULL, V varbinary(1024) DEFAULT NULL, PRIMARY KEY (K, Q, T) ) kv_attributes ='{"Hbase": {"TimeToLive": 1, "MaxVersions": 0}}' partition by key(K) partitions 3; -ERROR HY000: Incorrect arguments to max versions, which should greater than 0 +ERROR HY000: MaxVersions of hbase table must be greater than 0 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -46,7 +46,7 @@ T bigint NOT NULL, V varbinary(1024) DEFAULT NULL, PRIMARY KEY (K, Q, T) ) kv_attributes ='{"Hbase": {"TimeToLive": 0, "MaxVersions": 1}}' partition by key(K) partitions 3; -ERROR HY000: Incorrect arguments to time to live, which should greater than 0 +ERROR HY000: Time to live of hbase table must be greater than 0 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -62,7 +62,7 @@ T bigint NOT NULL, V varbinary(1024) DEFAULT NULL, PRIMARY KEY (K, Q, T) ) kv_attributes ='{"Hbase": {"TimeToLive": 0}, "Hbase": {"MaxVersions": 1}}' partition by key(K) partitions 3; -ERROR HY000: Invalid argument +ERROR 0A000: kv attributes with wrong format not supported CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -73,15 +73,15 @@ PRIMARY KEY (K, Q, T) alter table test_kv_attr kv_attributes '{"HColumnDescriptor": {"TimeToLive": 5, "MaxVersions": 2}}'; ERROR 0A000: kv attributes with wrong format not supported alter table test_kv_attr kv_attributes '{}'; -ERROR HY000: Invalid argument +ERROR 0A000: kv attributes with wrong format not supported alter table test_kv_attr kv_attributes ='{"Hbase": {"TimeToLive": 5, "MaxVersions": 2, "MinVersions": 1}}'; ERROR 0A000: kv attributes with wrong format not supported alter table test_kv_attr kv_attributes = '{"Hbase": {"MaxVersion": 2}}'; ERROR 0A000: kv attributes with wrong format not supported alter table test_kv_attr kv_attributes = '{"Hbase": {"TimeToLive": 1, "MaxVersions": 0}}'; -ERROR HY000: Incorrect arguments to max versions, which should greater than 0 +ERROR HY000: MaxVersions of hbase table must be greater than 0 alter table test_kv_attr kv_attributes ='{"Hbase": {"TimeToLive": 0, "MaxVersions": 1}}'; -ERROR HY000: Incorrect arguments to time to live, which should greater than 0 +ERROR HY000: Time to live of hbase table must be greater than 0 alter table test_kv_attr kv_attributes ='{"Hbase": {"MaxVersions": 4}}'; alter table test_kv_attr kv_attributes ='{"Hbase": {"TimeToLive": 10000000}}'; alter table test_kv_attr kv_attributes ='{"Hbase": {"MaxVersions": 4, "TimeToLive": 10000000}}'; @@ -108,13 +108,15 @@ test_ttl CREATE TABLE `test_ttl` ( ) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' REPLICA_NUM = 2 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 TTL = (c + INTERVAL 1 hour) drop table test_ttl; create table test_ttl(a int primary key, b varchar(1024), c varchar(1024)) TTL(c + INTERVAL 1 hour); -ERROR 0A000: Not supported feature or function +ERROR HY000: Column type of 'c' is not supported for TTL definition create table test_ttl(a int primary key, b varchar(1024), c date) TTL(c + INTERVAL 1 hour); -ERROR 0A000: Not supported feature or function +ERROR HY000: Column type of 'c' is not supported for TTL definition create table test_ttl(a int primary key, b varchar(1024), c time) TTL(c + INTERVAL 1 hour); -ERROR 0A000: Not supported feature or function +ERROR HY000: Column type of 'c' is not supported for TTL definition create table test_ttl(a int primary key, b varchar(1024)) TTL(now() + INTERVAL 1 hour); -ERROR 0A000: Not supported feature or function +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near '() + INTERVAL 1 hour)' at line 1 +create table test_ttl(a int primary key, b varchar(1024)) TTL(c + INTERVAL 1 hour); +ERROR HY000: TTL column 'c' not exists create table test_ttl(a int primary key, b varchar(1024), c timestamp, d datetime) TTL(c + INTERVAL 0 hour); show create table test_ttl; Table Create Table @@ -148,15 +150,17 @@ create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + I drop table test_ttl; create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVAL 1 year); alter table test_ttl drop column c; -ERROR 0A000: Not supported feature or function +ERROR 0A000: Modify/Change TTL column not supported alter table test_ttl change column c c2 datetime; -ERROR 0A000: Not supported feature or function +ERROR 0A000: Modify/Change TTL column not supported alter table test_ttl modify column c bigint; -ERROR 0A000: Not supported feature or function +ERROR 0A000: Modify/Change TTL column not supported alter table test_ttl modify column c datetime; -ERROR 0A000: Not supported feature or function +ERROR 0A000: Modify/Change TTL column not supported +alter table test_ttl ttl(d + INTERVAL 1 minute); +ERROR HY000: TTL column 'd' not exists alter table test_ttl add column d datetime, ttl(d + INTERVAL 1 minute); -ERROR 0A000: Not supported feature or function +ERROR HY000: TTL column 'd' not exists alter table test_ttl remove ttl; show create table test_ttl; Table Create Table @@ -167,5 +171,35 @@ test_ttl CREATE TABLE `test_ttl` ( PRIMARY KEY (`a`) ) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' REPLICA_NUM = 2 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 alter table test_ttl drop column c; +drop table test_ttl; +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVAL -1 year); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near '-1 year)' at line 1 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c +- INTERVAL -1 year); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near '- INTERVAL -1 year)' at line 1 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVA L -1 year); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near 'INTERVA L -1 year)' at line 1 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL('c' + INTERVAL 1 year); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near ''c' + INTERVAL 1 year)' at line 1 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVAL '1' year); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near ''1' year)' at line 1 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL( c + INTERVAL 1 year ); +drop table test_ttl; +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL( `c` + INTERVAL 1 year ); +alter table test_ttl TTL( c + INTERVAL 1 month ); +alter table test_ttl TTL( c + INTERVAL 1 day ); +alter table test_ttl TTL( c + INTERVAL 1 hour ); +alter table test_ttl TTL( `c` + INTERVAL 1 minute ); +alter table test_ttl TTL( `c` + INTERVAL 1 second ); +alter table test_ttl add column d datetime; +alter table test_ttl TTL( `d` + INTERVAL 1 second ); +show create table test_ttl; +Table Create Table +test_ttl CREATE TABLE `test_ttl` ( + `a` int(11) NOT NULL, + `b` varchar(1024) DEFAULT NULL, + `c` timestamp NULL DEFAULT NULL, + `d` datetime DEFAULT NULL, + PRIMARY KEY (`a`) +) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' REPLICA_NUM = 2 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0 TTL = (`d` + INTERVAL 1 second) drop table test_kv_attr; drop table test_ttl; diff --git a/tools/deploy/mysql_test/test_suite/table_api/t/ttl_ddl.test b/tools/deploy/mysql_test/test_suite/table_api/t/ttl_ddl.test index 6583519f06..622d872d29 100644 --- a/tools/deploy/mysql_test/test_suite/table_api/t/ttl_ddl.test +++ b/tools/deploy/mysql_test/test_suite/table_api/t/ttl_ddl.test @@ -8,6 +8,7 @@ connection default; ### test kv_attributes syntax --disable_warnings drop table if exists test_kv_attr; +--enable_warnings --error 1235 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, @@ -17,7 +18,7 @@ CREATE TABLE test_kv_attr( PRIMARY KEY (K, Q, T) ) kv_attributes ='{"HColumnDescriptor": {"TimeToLive": 5, "MaxVersions": 2}}' partition by key(K) partitions 3; ---error 1210 +--error 1235 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -44,7 +45,7 @@ CREATE TABLE test_kv_attr( PRIMARY KEY (K, Q, T) ) kv_attributes ='{"Hbase": {"TimeToLive": 5, "MaxVersions": 2, "MinVersions": 1}}' partition by key(K) partitions 3; ---error 1210 +--error 10508 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -53,7 +54,7 @@ CREATE TABLE test_kv_attr( PRIMARY KEY (K, Q, T) ) kv_attributes ='{"Hbase": {"TimeToLive": 1, "MaxVersions": 0}}' partition by key(K) partitions 3; ---error 1210 +--error 10507 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -71,7 +72,7 @@ CREATE TABLE test_kv_attr( PRIMARY KEY (K, Q, T) ) kv_attributes ='{"TimeSeries": {}}' partition by key(K) partitions 3; ---error 1210 +--error 1235 CREATE TABLE test_kv_attr( K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, @@ -90,15 +91,15 @@ CREATE TABLE test_kv_attr( --error 1235 alter table test_kv_attr kv_attributes '{"HColumnDescriptor": {"TimeToLive": 5, "MaxVersions": 2}}'; ---error 1210 +--error 1235 alter table test_kv_attr kv_attributes '{}'; --error 1235 alter table test_kv_attr kv_attributes ='{"Hbase": {"TimeToLive": 5, "MaxVersions": 2, "MinVersions": 1}}'; --error 1235 alter table test_kv_attr kv_attributes = '{"Hbase": {"MaxVersion": 2}}'; ---error 1210 +--error 10508 alter table test_kv_attr kv_attributes = '{"Hbase": {"TimeToLive": 1, "MaxVersions": 0}}'; ---error 1210 +--error 10507 alter table test_kv_attr kv_attributes ='{"Hbase": {"TimeToLive": 0, "MaxVersions": 1}}'; alter table test_kv_attr kv_attributes ='{"Hbase": {"MaxVersions": 4}}'; @@ -110,6 +111,9 @@ alter table test_kv_attr kv_attributes ='{"Hbase": {}}'; ### test ttl syntax --disable_warnings drop table if exists test_ttl; +--enable_warnings + +# test show create table create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVAL 1 hour); show create table test_ttl; drop table test_ttl; @@ -118,14 +122,16 @@ show create table test_ttl; drop table test_ttl; ## not supported data type ---error 1235 +--error 10503 create table test_ttl(a int primary key, b varchar(1024), c varchar(1024)) TTL(c + INTERVAL 1 hour); ---error 1235 +--error 10503 create table test_ttl(a int primary key, b varchar(1024), c date) TTL(c + INTERVAL 1 hour); ---error 1235 +--error 10503 create table test_ttl(a int primary key, b varchar(1024), c time) TTL(c + INTERVAL 1 hour); ---error 1235 +--error 1064 create table test_ttl(a int primary key, b varchar(1024)) TTL(now() + INTERVAL 1 hour); +--error 10502 +create table test_ttl(a int primary key, b varchar(1024)) TTL(c + INTERVAL 1 hour); ## alter table ttl create table test_ttl(a int primary key, b varchar(1024), c timestamp, d datetime) TTL(c + INTERVAL 0 hour); @@ -156,12 +162,43 @@ alter table test_ttl change column c c2 datetime; alter table test_ttl modify column c bigint; --error 1235 alter table test_ttl modify column c datetime; ---error 1235 +--error 10502 +alter table test_ttl ttl(d + INTERVAL 1 minute); +--error 10502 alter table test_ttl add column d datetime, ttl(d + INTERVAL 1 minute); +# remove ttl definition and add column alter table test_ttl remove ttl; show create table test_ttl; alter table test_ttl drop column c; +drop table test_ttl; + +# test ttl definition parse +--error 1064 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVAL -1 year); +--error 1064 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c +- INTERVAL -1 year); +--error 1064 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVA L -1 year); +--error 1064 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL('c' + INTERVAL 1 year); +--error 1064 +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL(c + INTERVAL '1' year); + +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL( c + INTERVAL 1 year ); +drop table test_ttl; +create table test_ttl(a int primary key, b varchar(1024), c timestamp) TTL( `c` + INTERVAL 1 year ); +alter table test_ttl TTL( c + INTERVAL 1 month ); +alter table test_ttl TTL( c + INTERVAL 1 day ); +alter table test_ttl TTL( c + INTERVAL 1 hour ); +alter table test_ttl TTL( `c` + INTERVAL 1 minute ); +alter table test_ttl TTL( `c` + INTERVAL 1 second ); + +# test change ttl column +alter table test_ttl add column d datetime; +alter table test_ttl TTL( `d` + INTERVAL 1 second ); +show create table test_ttl; + drop table test_kv_attr; drop table test_ttl;