diff --git a/deps/oblib/src/lib/hash/ob_hashmap.h b/deps/oblib/src/lib/hash/ob_hashmap.h index 98a1f74e7a..c99ae76d44 100644 --- a/deps/oblib/src/lib/hash/ob_hashmap.h +++ b/deps/oblib/src/lib/hash/ob_hashmap.h @@ -245,7 +245,25 @@ public: } return ret; }; - + // 该原子操作在bucket上添加的写锁, + // 如果节点存在，调用 callback 进行修改，如果节点不存在，插入该节点 + // + // 返回值： + // OB_SUCCESS 表示成功 + // 其它 表示出错 + template + int set_or_update(const _key_type &key, const _value_type &value, + _callback &callback) + { + int ret = OB_SUCCESS; + pair_type pair; + if (OB_FAIL(pair.init(key, value))) { + HASH_WRITE_LOG(HASH_WARNING, "init pair failed, ret=%d", ret); + } else { + ret = ht_.set_or_update(key, pair, callback); + } + return ret; + }; // erase key value pair if pred is met // thread safe erase, will add write lock to the bucket // return value: @@ -267,7 +285,6 @@ public: } return ret; } - template int serialization(_archive &archive) { diff --git a/deps/oblib/src/lib/hash/ob_hashtable.h b/deps/oblib/src/lib/hash/ob_hashtable.h index 8e91c7ec2d..e2e1bd83bf 100644 --- a/deps/oblib/src/lib/hash/ob_hashtable.h +++ b/deps/oblib/src/lib/hash/ob_hashtable.h @@ -1311,6 +1311,19 @@ public: return read_atomic(key, callback, preproc_); } + // 该原子操作在bucket上添加的写锁, + // 如果节点存在，调用 callback 进行修改，如果节点不存在，插入该节点 + // + // 返回值： + // OB_SUCCESS 表示成功 + // 其它 表示出错 + template + int set_or_update(const _key_type &key, const _value_type &value, + _callback &callback) + { + return set_or_update(key, value, callback, preproc_); + } + int erase_refactored(const _key_type &key, _value_type *value = NULL) { int ret = OB_SUCCESS; @@ -1597,6 +1610,40 @@ public: return ret; } + // 不存在就插入，存在就调用 callback 修改 + // 该原子操作在bucket上添加的写锁 + template + int set_or_update(const _key_type &key, const _value_type &value, + _callback &callback, _preproc &preproc) + { + int ret = OB_SUCCESS; + uint64_t hash_value; + if (OB_UNLIKELY(!inited(buckets_)) || OB_UNLIKELY(NULL == allocer_)) { + HASH_WRITE_LOG(HASH_WARNING, "hashtable not init"); + ret = OB_NOT_INIT; + } else if (OB_FAIL(hashfunc_(key, hash_value))) { + HASH_WRITE_LOG(HASH_WARNING, "hash key failed, ret = %d", ret); + } else { + int64_t bucket_pos = hash_value % bucket_num_; + hashbucket &bucket = buckets_[bucket_pos]; + bucket_lock_cond blc(bucket); + writelocker locker(blc.lock()); + hashnode *node = bucket.node; + while (NULL != node) { + if (equal_(getkey_(node->data), key)) { + callback(preproc(node->data)); + break; + } else { + node = node->next; + } + } + if (NULL == node) { + ret = internal_set(bucket, value, false); + } + } + + return ret; + } private: _bucket_allocer default_bucket_allocer_; _allocer *allocer_; diff --git a/deps/oblib/src/lib/lock/ob_latch.h b/deps/oblib/src/lib/lock/ob_latch.h index 70970eb53c..12069a4eb1 100644 --- a/deps/oblib/src/lib/lock/ob_latch.h +++ b/deps/oblib/src/lib/lock/ob_latch.h @@ -231,6 +231,7 @@ public: inline bool is_wrlocked() const; inline bool is_wrlocked_by(const uint32_t *puid = NULL) const; inline uint32_t get_wid() const; + inline uint32_t get_rdcnt() const; int64_t to_string(char* buf, const int64_t buf_len) const; void enable_record_stat(bool enable) { record_stat_ = enable; } bool need_record_stat() const { return record_stat_; } @@ -540,6 +541,12 @@ inline uint32_t ObLatch::get_wid() const return (0 == (lock & WRITE_MASK)) ? 0 : (lock & ~(WAIT_MASK | WRITE_MASK)); } +inline uint32_t ObLatch::get_rdcnt() const +{ + uint32_t lock = ATOMIC_LOAD(&lock_); + return (0 == (lock & WRITE_MASK)) ? (lock & ~(WAIT_MASK | WRITE_MASK)) : 0; +} + inline int ObLatch::LowTryRDLock::operator()(volatile uint32_t *latch, const uint32_t lock, const uint32_t uid, bool &conflict) { diff --git a/deps/oblib/unittest/lib/hash/test_hashmap.cpp b/deps/oblib/unittest/lib/hash/test_hashmap.cpp index 7b15bd1fd7..c0afa7f5ee 100644 --- a/deps/oblib/unittest/lib/hash/test_hashmap.cpp +++ b/deps/oblib/unittest/lib/hash/test_hashmap.cpp @@ -45,6 +45,22 @@ class CallBack HashValue v_; }; +class Predicate +{ + public: + bool operator () (HashMapPair &v) + { + return v.second >= min_value_; + }; + void set_min_value(HashValue v) + { + min_value_ = v; + }; + private: + HashValue min_value_; +}; + + TEST(TestObHashMap, create) { ObHashMap hm; @@ -274,6 +290,60 @@ TEST(TestObHashMap, atomic) EXPECT_EQ(value_update, value_tmp); } +TEST(TestObHashMap, set_or_update) +{ + ObHashMap hm; + uint64_t key = 1; + uint64_t value = 100; + CallBack callback; + HashValue value_tmp; + + // 没有create + EXPECT_EQ(OB_NOT_INIT, hm.set_or_update(key, value, callback)); + hm.create(cal_next_prime(gHashItemNum), ObModIds::OB_HASH_BUCKET); + + callback.set_v(value); + EXPECT_EQ(OB_HASH_NOT_EXIST, hm.get_refactored(key, value_tmp)); + EXPECT_EQ(OB_SUCCESS, hm.set_or_update(key, value, callback)); + EXPECT_EQ(OB_SUCCESS, hm.get_refactored(key, value_tmp)); + EXPECT_EQ(value, value_tmp); + + uint64_t value_update = 3000; + callback.set_v(value_update); + EXPECT_EQ(OB_SUCCESS, hm.set_or_update(key, value, callback)); + EXPECT_EQ(OB_SUCCESS, hm.get_refactored(key, value_tmp)); + EXPECT_EQ(value_update, value_tmp); +} + +TEST(TestObHashMap, erase_if) +{ + ObHashMap hm; + uint64_t key = 1; + uint64_t value = 100; + Predicate pred; + HashValue value_tmp; + bool is_erased = true; + + // 没有create + EXPECT_EQ(OB_NOT_INIT, hm.erase_if(key, pred, is_erased)); + hm.create(cal_next_prime(gHashItemNum), ObModIds::OB_HASH_BUCKET); + + pred.set_min_value(value + 1); + EXPECT_EQ(OB_HASH_NOT_EXIST, hm.get_refactored(key, value_tmp)); + EXPECT_EQ(OB_SUCCESS, hm.set_refactored(key, value)); + EXPECT_EQ(OB_SUCCESS, hm.erase_if(key, pred, is_erased, &value_tmp)); + EXPECT_EQ(false, is_erased); + EXPECT_EQ(OB_SUCCESS, hm.get_refactored(key, value_tmp)); + EXPECT_EQ(value, value_tmp); + + pred.set_min_value(value); + value_tmp = 0; + EXPECT_EQ(OB_SUCCESS, hm.erase_if(key, pred, is_erased, &value_tmp)); + EXPECT_EQ(true, is_erased); + EXPECT_EQ(value, value_tmp); + EXPECT_EQ(OB_HASH_NOT_EXIST, hm.get_refactored(key, value_tmp)); +} + struct GAllocator { void *alloc(const int64_t sz) diff --git a/mittest/mtlenv/CMakeLists.txt b/mittest/mtlenv/CMakeLists.txt index f13e949630..8e476d4a9a 100644 --- a/mittest/mtlenv/CMakeLists.txt +++ b/mittest/mtlenv/CMakeLists.txt @@ -3,6 +3,7 @@ sql_unittest(test_session_serde) storage_unittest(test_tx_data_table) #storage_unittest(test_multi_tenant test_multi_tenant.cpp) storage_unittest(test_buffer_ctx_node test_buffer_ctx_node.cpp) +ob_unittest(test_htable_lock test_htable_lock.cpp) add_subdirectory(storage) add_subdirectory(tablelock) diff --git a/mittest/mtlenv/mock_tenant_module_env.h b/mittest/mtlenv/mock_tenant_module_env.h index eba7d5f63b..0b36ecb0bc 100644 --- a/mittest/mtlenv/mock_tenant_module_env.h +++ b/mittest/mtlenv/mock_tenant_module_env.h @@ -84,6 +84,7 @@ #include "storage/tx/wrs/ob_tenant_weak_read_service.h" #include "logservice/palf/log_define.h" #include "storage/high_availability/ob_rebuild_service.h" +#include "observer/table/ob_htable_lock_mgr.h" namespace oceanbase { @@ -700,6 +701,7 @@ int MockTenantModuleEnv::init() MTL_BIND2(server_obj_pool_mtl_new, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy); MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy); MTL_BIND2(mtl_new_default, ObRebuildService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); + MTL_BIND(table::ObHTableLockMgr::mtl_init, table::ObHTableLockMgr::mtl_destroy); MTL_BIND2(mtl_new_default, omt::ObTenantSrs::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); } if (OB_FAIL(ret)) { diff --git a/mittest/mtlenv/test_htable_lock.cpp b/mittest/mtlenv/test_htable_lock.cpp new file mode 100644 index 0000000000..4266d37f49 --- /dev/null +++ b/mittest/mtlenv/test_htable_lock.cpp @@ -0,0 +1,461 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define private public +#define protected public +#include "observer/table/ob_htable_lock_mgr.h" +#include "lib/utility/ob_test_util.h" +#include "share/ob_thread_pool.h" +#include "mock_tenant_module_env.h" + +using namespace oceanbase; +using namespace oceanbase::common; +using namespace oceanbase::table; +using namespace oceanbase::transaction; +using namespace oceanbase::share; + +class TestHTableLock: public ::testing::Test +{ +public: + TestHTableLock() {} + static void SetUpTestCase() + { + EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); + } + static void TearDownTestCase() + { + MockTenantModuleEnv::get_instance().destroy(); + } + void SetUp() + { + ASSERT_TRUE(MockTenantModuleEnv::get_instance().is_inited()); + } + virtual ~TestHTableLock() {} +private: + DISALLOW_COPY_AND_ASSIGN(TestHTableLock); +}; + +class TestConcurrentHTableLock : public ObThreadPool +{ +public: + explicit TestConcurrentHTableLock(ObHTableLockMgr* mgr, const int64_t thread_cnt, const uint64_t table_id, const ObString &key, ObHTableLockMode mode, bool can_retry = false) + : mgr(mgr), + thread_cnt_(thread_cnt), + table_id_(table_id), + key_(key), + mode_(mode), + fake_addr_(), + can_retry_(can_retry) + { + set_thread_count(thread_cnt); + } + virtual ~TestConcurrentHTableLock() {}; + virtual void run1(); + virtual void process() = 0; + +private: + ObHTableLockMgr* mgr; + const int64_t thread_cnt_; + const uint64_t table_id_; + const ObString key_; + ObHTableLockMode mode_; + ObAddr fake_addr_; + bool can_retry_; +}; + +void TestConcurrentHTableLock::run1() +{ + ObArenaAllocator alloc; + ObTransID fake_tx_id; + ObString local_key; + ObHTableLockHandle *lock_handle = nullptr; + ASSERT_EQ(OB_SUCCESS, ob_write_string(alloc, key_, local_key)); + ASSERT_EQ(OB_SUCCESS, mgr->acquire_handle(fake_tx_id, lock_handle)); + int ret = OB_SUCCESS; + ret = mgr->lock_row(table_id_, local_key, mode_, *lock_handle); + if (can_retry_) { + while (ret == OB_TRY_LOCK_ROW_CONFLICT) { + ret = mgr->lock_row(table_id_, local_key, mode_, *lock_handle); + } + } + ASSERT_EQ(OB_SUCCESS, ret); + process(); + ASSERT_EQ(OB_SUCCESS, mgr->release_handle(*lock_handle)); +} + +class TestConcurrentHTableSLock : public TestConcurrentHTableLock +{ +public: + TestConcurrentHTableSLock(ObHTableLockMgr* mgr, const int64_t thread_cnt, const uint64_t table_id, const ObString &key, bool can_retry = false) + : TestConcurrentHTableLock(mgr, thread_cnt, table_id, key, ObHTableLockMode::SHARED, can_retry) + {} + virtual ~TestConcurrentHTableSLock() {}; + virtual void process() override; +}; + +void TestConcurrentHTableSLock::process() +{ + usleep(1*1000); // 1ms, for processing +} + +class TestConcurrentHTableXLock : public TestConcurrentHTableLock +{ +public: + TestConcurrentHTableXLock(ObHTableLockMgr* mgr, const int64_t thread_cnt, const uint64_t table_id, const ObString &key) + : TestConcurrentHTableLock(mgr, thread_cnt, table_id, key, ObHTableLockMode::EXCLUSIVE, true), + value_(0) + {} + virtual ~TestConcurrentHTableXLock() {}; + virtual void process() override; + void check_result(); + +private: + uint64_t value_; +}; + +void TestConcurrentHTableXLock::process() +{ + uint64_t value = value_; + usleep(1*1000); // 1ms, for processing + value_ = value + 1; +} + +void TestConcurrentHTableXLock::check_result() +{ + ASSERT_EQ(value_, thread_cnt_); +} + +TEST_F(TestHTableLock, basic_test) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ObTransID fake_tx_id; + ObHTableLockHandle *lock_handle = nullptr; + uint64_t fake_table_id = 1; + ObString fake_key = ObString::make_string("k1"); + // 1. acquire htable lock handle for a running tx + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id, lock_handle)); + ASSERT_TRUE(lock_handle != nullptr); + // 2. add htable lock on htable row in demand during tx + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle)); + // 3. release htable lock handle after transaction commit or rollback + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle)); +} + +TEST_F(TestHTableLock, lock_different_keys) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ObTransID fake_tx_id; + ObHTableLockHandle *lock_handle = nullptr; + uint64_t fake_table_id = 1; + ObString fake_key1 = ObString::make_string("k1"); + ObString fake_key2 = ObString::make_string("k2"); + ObHTableLockKey lock_key1(fake_table_id, fake_key1); + ObHTableLockKey lock_key2(fake_table_id, fake_key2); + ObHTableLockKey *tmp_lock_key = nullptr; + ObHTableLock *tmp_lock = nullptr; + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id, lock_handle)); + ASSERT_TRUE(lock_handle != nullptr); + ASSERT_EQ(fake_tx_id, lock_handle->tx_id_); + ASSERT_EQ(nullptr, lock_handle->lock_nodes_); + + // lock fake_key1 in share mode + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key1, ObHTableLockMode::SHARED, *lock_handle)); + // check lock handle + ASSERT_EQ(ObHTableLockMode::SHARED, lock_handle->lock_nodes_->lock_mode_); + ASSERT_EQ(fake_table_id, lock_handle->lock_nodes_->lock_key_->table_id_); + ASSERT_EQ(fake_key1, lock_handle->lock_nodes_->lock_key_->key_); + // check lock map + ASSERT_EQ(1, HTABLE_LOCK_MGR->lock_map_.size()); + ObHTableLockMgr::ObHTableLockMap::const_iterator it = HTABLE_LOCK_MGR->lock_map_.begin(); + tmp_lock_key = it->first; + ASSERT_TRUE(tmp_lock_key != nullptr); + ASSERT_EQ(lock_key1, *tmp_lock_key); + ASSERT_TRUE(tmp_lock_key->key_.ptr() != fake_key1.ptr()); // ensure it's deep copy + ASSERT_EQ(lock_handle->lock_nodes_->lock_key_->key_.ptr(), tmp_lock_key->key_.ptr()); + tmp_lock = it->second; + ASSERT_TRUE(tmp_lock->is_locked()); + ASSERT_TRUE(tmp_lock->is_rdlocked()); + ASSERT_TRUE(tmp_lock->can_escalate_lock()); + ASSERT_TRUE(!tmp_lock->is_wrlocked()); + + // lock fake_key2 in exclusive mode + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key2, ObHTableLockMode::EXCLUSIVE, *lock_handle)); + // check lock handle + ASSERT_EQ(ObHTableLockMode::EXCLUSIVE, lock_handle->lock_nodes_->lock_mode_); + ASSERT_EQ(fake_table_id, lock_handle->lock_nodes_->lock_key_->table_id_); + ASSERT_EQ(fake_key2, lock_handle->lock_nodes_->lock_key_->key_); + ASSERT_EQ(ObHTableLockMode::SHARED, lock_handle->lock_nodes_->next_->lock_mode_); + ASSERT_EQ(fake_table_id, lock_handle->lock_nodes_->next_->lock_key_->table_id_); + ASSERT_EQ(fake_key1, lock_handle->lock_nodes_->next_->lock_key_->key_); + // check lock map + tmp_lock = nullptr; + ASSERT_EQ(2, HTABLE_LOCK_MGR->lock_map_.size()); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_map_.get_refactored(&lock_key1, tmp_lock)); + ASSERT_TRUE(tmp_lock != nullptr); + ASSERT_TRUE(tmp_lock->is_locked()); + ASSERT_TRUE(tmp_lock->is_rdlocked()); + ASSERT_TRUE(tmp_lock->can_escalate_lock()); + ASSERT_TRUE(!tmp_lock->is_wrlocked()); + tmp_lock = nullptr; + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_map_.get_refactored(&lock_key2, tmp_lock)); + ASSERT_TRUE(tmp_lock != nullptr); + ASSERT_TRUE(tmp_lock->is_locked()); + ASSERT_TRUE(!tmp_lock->is_rdlocked()); + ASSERT_TRUE(!tmp_lock->can_escalate_lock()); + ASSERT_TRUE(tmp_lock->is_wrlocked()); + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle)); +} + +TEST_F(TestHTableLock, share_share) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ObTransID fake_tx_id1(1); + ObTransID fake_tx_id2(2); + ObHTableLockHandle *lock_handle1 = nullptr; + ObHTableLockHandle *lock_handle2 = nullptr; + uint64_t fake_table_id = 1; + ObString fake_key = ObString::make_string("k1"); + ObHTableLockKey lock_key(fake_table_id, fake_key); + ObHTableLockKey *tmp_lock_key = nullptr; + ObHTableLock *tmp_lock = nullptr; + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id1, lock_handle1)); + ASSERT_TRUE(lock_handle1 != nullptr); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id2, lock_handle2)); + ASSERT_TRUE(lock_handle2 != nullptr); + + // two lock handles try to add share lock on one row concurrently return success + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle1)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle2)); + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle1)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle2)); +} + +TEST_F(TestHTableLock, share_exclusive) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ObTransID fake_tx_id1(1); + ObTransID fake_tx_id2(2); + ObHTableLockHandle *lock_handle1 = nullptr; + ObHTableLockHandle *lock_handle2 = nullptr; + uint64_t fake_table_id = 1; + ObString fake_key = ObString::make_string("k1"); + ObHTableLockKey lock_key(fake_table_id, fake_key); + ObHTableLockKey *tmp_lock_key = nullptr; + ObHTableLock *tmp_lock = nullptr; + + // acquire lock handle1 and lock handle2 + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id1, lock_handle1)); + ASSERT_TRUE(lock_handle1 != nullptr); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id2, lock_handle2)); + ASSERT_TRUE(lock_handle2 != nullptr); + + // lock handle1 add share lock, and lock handle2 add exclusive lock + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle1)); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + + // release lock handle1 and handle2 and acquire again + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle1)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle2)); + lock_handle1 = nullptr; + lock_handle2 = nullptr; + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id1, lock_handle1)); + ASSERT_TRUE(lock_handle1 != nullptr); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id2, lock_handle2)); + ASSERT_TRUE(lock_handle2 != nullptr); + + // lock handle1 add exclusive lock, and lock handle2 add share lock + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle1)); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle2)); + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle1)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle2)); +} + +TEST_F(TestHTableLock, exclusive_exclusive) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ObTransID fake_tx_id1(1); + ObTransID fake_tx_id2(2); + ObHTableLockHandle *lock_handle1 = nullptr; + ObHTableLockHandle *lock_handle2 = nullptr; + uint64_t fake_table_id = 1; + ObString fake_key = ObString::make_string("k1"); + ObHTableLockKey lock_key(fake_table_id, fake_key); + ObHTableLockKey *tmp_lock_key = nullptr; + ObHTableLock *tmp_lock = nullptr; + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id1, lock_handle1)); + ASSERT_TRUE(lock_handle1 != nullptr); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id2, lock_handle2)); + ASSERT_TRUE(lock_handle2 != nullptr); + + // lock handle1 add exclusive lock, and lock handle2 add exclusive lock + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle1)); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + + // release lock handle1 + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle1)); + // lock handle2 try to add exclusive lock again + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + // release lock handle2 + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle2)); +} + + +TEST_F(TestHTableLock, repeat_lock_one_row) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ObTransID fake_tx_id; + ObHTableLockHandle *lock_handle = nullptr; + ObHTableLockHandle *lock_handle2 = nullptr; + uint64_t fake_table_id = 1; + ObString fake_key = ObString::make_string("k1"); + ObHTableLockKey lock_key(fake_table_id, fake_key); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id, lock_handle)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id, lock_handle2)); + ASSERT_TRUE(lock_handle != nullptr); + ASSERT_TRUE(lock_handle2 != nullptr); + + // add intial share lock + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle)); + + // 1. share -> share, noting changed + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle)); + ASSERT_TRUE(lock_handle->lock_nodes_ != nullptr); + ASSERT_EQ(ObHTableLockMode::SHARED, lock_handle->lock_nodes_->lock_mode_); + ASSERT_EQ(nullptr, lock_handle->lock_nodes_->next_); + ASSERT_EQ(lock_key, *lock_handle->lock_nodes_->lock_key_); + // the others try to add lock + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle2)); + + // 2. share -> exclusive, will escalate share lock to exclusive lock when no one else holds the lock + // lock handle2 hold the share lock too, escalate lock failed + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle2)); + lock_handle2 = nullptr; + // lock handle2 release the shared lock, escalate lock success + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle)); + ASSERT_EQ(ObHTableLockMode::EXCLUSIVE, lock_handle->lock_nodes_->lock_mode_); + ASSERT_EQ(nullptr, lock_handle->lock_nodes_->next_); + ASSERT_EQ(lock_key, *lock_handle->lock_nodes_->lock_key_); + // the others try to add lock + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->acquire_handle(fake_tx_id, lock_handle2)); + ASSERT_TRUE(lock_handle2 != nullptr); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle2)); + + // 3. exclusive -> share, still hold exclusive lock + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle)); + ASSERT_TRUE(lock_handle->lock_nodes_ != nullptr); + ASSERT_EQ(ObHTableLockMode::EXCLUSIVE, lock_handle->lock_nodes_->lock_mode_); + ASSERT_EQ(nullptr, lock_handle->lock_nodes_->next_); + ASSERT_EQ(lock_key, *lock_handle->lock_nodes_->lock_key_); + // the others try to add lock + ASSERT_TRUE(lock_handle->lock_nodes_ != nullptr); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle2)); + + // 4. exclusive -> exclusive, nothing changed + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle)); + ASSERT_TRUE(lock_handle->lock_nodes_ != nullptr); + ASSERT_EQ(ObHTableLockMode::EXCLUSIVE, lock_handle->lock_nodes_->lock_mode_); + ASSERT_EQ(nullptr, lock_handle->lock_nodes_->next_); + ASSERT_EQ(lock_key, *lock_handle->lock_nodes_->lock_key_); + // the others try to add lock + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::EXCLUSIVE, *lock_handle2)); + ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, HTABLE_LOCK_MGR->lock_row(fake_table_id, fake_key, ObHTableLockMode::SHARED, *lock_handle2)); + + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle)); + ASSERT_EQ(OB_SUCCESS, HTABLE_LOCK_MGR->release_handle(*lock_handle2)); +} + +TEST_F(TestHTableLock, concurrent_shared_lock) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + const uint64_t thread_cnt = 2048; + const uint64_t fake_table_id = 1; + ObString key = ObString::make_string("k1"); + const int64_t start = ObTimeUtility::current_time(); + TestConcurrentHTableSLock multi_thread(HTABLE_LOCK_MGR, thread_cnt, fake_table_id, key); + ASSERT_EQ(OB_SUCCESS, multi_thread.start()); + multi_thread.wait(); + const int64_t duration = ObTimeUtility::current_time() - start; + printf("thread_cnt: %ld, time elapsed: %ldms\n", thread_cnt, duration/1000); +} + +TEST_F(TestHTableLock, concurrent_exclusive_lock) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + const uint64_t thread_cnt = 1024; + const uint64_t fake_table_id = 1; + ObString key = ObString::make_string("k1"); + const int64_t start = ObTimeUtility::current_time(); + TestConcurrentHTableXLock multi_thread(HTABLE_LOCK_MGR, thread_cnt, fake_table_id, key); + ASSERT_EQ(OB_SUCCESS, multi_thread.start()); + multi_thread.wait(); + multi_thread.check_result(); + const int64_t duration = ObTimeUtility::current_time() - start; + printf("thread_cnt: %ld, time elapsed: %ldms\n", thread_cnt, duration/1000); +} + +TEST_F(TestHTableLock, concurrent_exclusive_shared_lock) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + const uint64_t xthread_cnt = 1024; + const uint64_t sthread_cnt = 1024; + const uint64_t fake_table_id = 1; + ObString key = ObString::make_string("k1"); + const int64_t start = ObTimeUtility::current_time(); + TestConcurrentHTableXLock multi_xthread(HTABLE_LOCK_MGR, xthread_cnt, fake_table_id, key); + TestConcurrentHTableSLock multi_sthread1(HTABLE_LOCK_MGR, sthread_cnt, fake_table_id, key, true); + ASSERT_EQ(OB_SUCCESS, multi_xthread.start()); + ASSERT_EQ(OB_SUCCESS, multi_sthread1.start()); + sleep(3); + TestConcurrentHTableSLock multi_sthread2(HTABLE_LOCK_MGR, sthread_cnt, fake_table_id, key, true); + ASSERT_EQ(OB_SUCCESS, multi_sthread2.start()); + sleep(3); + TestConcurrentHTableSLock multi_sthread3(HTABLE_LOCK_MGR, sthread_cnt, fake_table_id, key, true); + ASSERT_EQ(OB_SUCCESS, multi_sthread3.start()); + multi_xthread.wait(); + multi_sthread1.wait(); + multi_sthread2.wait(); + multi_sthread3.wait(); + multi_xthread.check_result(); + const int64_t duration = ObTimeUtility::current_time() - start; + printf("exclusive thread cnt: %ld, shared thread cnt: %ld, time elapsed: %ldms\n", + xthread_cnt, sthread_cnt, duration/1000); +} + +// NOTE: Put this test at the end of the whole test case to check whether memory leak exists or not +TEST_F(TestHTableLock, check_mem) +{ + EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); + ASSERT_EQ(0, HTABLE_LOCK_MGR->allocator_.normal_used_); + ASSERT_EQ(0, HTABLE_LOCK_MGR->allocator_.special_total_); + ASSERT_EQ(1, HTABLE_LOCK_MGR->allocator_.current_using_->ref_count_); + ASSERT_EQ(0, HTABLE_LOCK_MGR->allocator_.special_page_list_.size_); + ASSERT_EQ(0, HTABLE_LOCK_MGR->allocator_.using_page_list_.size_); +} + +int main(int argc, char **argv) +{ + OB_LOGGER.set_log_level("INFO"); + OB_LOGGER.set_file_name("test_htable_lock.log", true); + ::testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/src/libtable/test/ob_batch_execute_test.cpp b/src/libtable/test/ob_batch_execute_test.cpp index 30d027cda4..b69e47e5aa 100644 --- a/src/libtable/test/ob_batch_execute_test.cpp +++ b/src/libtable/test/ob_batch_execute_test.cpp @@ -2827,7 +2827,7 @@ fprintf(stderr, "Case 1: reverse scan set max version only\n"); TEST_F(TestBatchExecute, hcolumn_desc) { ObHColumnDescriptor column_desc; - ObString str = ObString::make_string("{\"HColumnDescriptor\": {\"TimeToLive\": 3600}}"); + ObString str = ObString::make_string("{\"Hbase\": {\"TimeToLive\": 3600}}"); ASSERT_EQ(OB_SUCCESS, column_desc.from_string(str)); fprintf(stderr, "ttl=%d\n", column_desc.get_time_to_live()); ASSERT_EQ(3600, column_desc.get_time_to_live()); @@ -8382,7 +8382,8 @@ TEST_F(TestBatchExecute, htable_query_and_mutate) ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(1, affected_rows); } else { - ASSERT_EQ(OB_NOT_SUPPORTED, ret); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, affected_rows); // affected_rows should be 0 when check failed } } // end for @@ -8399,8 +8400,9 @@ TEST_F(TestBatchExecute, htable_query_and_mutate) range.border_flag_.unset_inclusive_end(); query.clear_scan_range(); ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - htable_filter.clear_columns(); + htable_filter.reset(); ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string("cq110"))); + htable_filter.set_valid(true); ObTableEntityIterator *iter = nullptr; ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); @@ -8498,9 +8500,10 @@ TEST_F(TestBatchExecute, htable_query_and_mutate) range.border_flag_.unset_inclusive_end(); query.clear_scan_range(); ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - htable_filter.clear_columns(); + htable_filter.reset(); ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string("cq0"))); htable_filter.set_max_versions(1); + htable_filter.set_valid(true); ObTableEntityIterator *iter = nullptr; ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); const ObITableEntity *result_entity = NULL; @@ -8784,6 +8787,110 @@ TEST_F(TestBatchExecute, htable_increment_empty) the_table = NULL; } +TEST_F(TestBatchExecute, htable_increment_multi_thread) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_increment"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + + // 1. execute delete first + const char* rowkey = "row2"; + const char *qualifier = "cq1"; + ObObj key1, key2, key3, value; + entity = entity_factory.alloc(); + key1.set_varbinary(ObString::make_string(rowkey)); + key2.set_varbinary(ObString::make_string(qualifier)); + key3.set_int(-INT64_MAX); // delete all versions + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); + ObTableBatchOperationResult del_result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, del_result)); + //////////////////////////////////////////////////////////////// + // 2. fill query and mutations + // query: query with row0 and htable filter for column cq0 + // mutation: row2, cq1, 1, 1 + ObTableQueryAndMutate query_and_mutate; + ObTableQuery &query = query_and_mutate.get_query(); + ObTableBatchOperation &mutations = query_and_mutate.get_mutations(); + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); + ObHTableFilter &htable_filter = query.htable_filter(); + htable_filter.add_column(ObString::make_string(qualifier)); + ASSERT_EQ(1, htable_filter.get_max_versions()); + htable_filter.set_valid(true); + mutations.reset(); + entity->reset(); + key3.set_int(INT64_MAX); // latest time + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + char inc_value[8]; + ObHTableUtils::int64_to_java_bytes(1, inc_value); + ObString inc_str(8, inc_value); + value.set_varbinary(inc_str); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, mutations.increment(*entity)); + // 3. execute increment and check result + auto task = [&](ObTable* one_table, const ObTableQueryAndMutate &query_and_mutate, uint64_t inc_times) { + ObTableQueryAndMutateResult inc_result; + for (int i = 0; i < inc_times; i++) { + ASSERT_EQ(OB_SUCCESS, one_table->execute_query_and_mutate(query_and_mutate, inc_result)); + ASSERT_EQ(1, inc_result.affected_rows_); + } + }; + constexpr uint64_t thread_num = 30; + constexpr uint64_t inc_times = 50; + std::vector threads; + time_t start_time = time(NULL); + printf("begin to run tasks, thread_num: %ld\n", thread_num); + for (uint64_t i = 0; i < thread_num; ++i) { + std::thread t(task, the_table, query_and_mutate, inc_times); + threads.push_back(std::move(t)); + } + for (uint64_t i = 0; i < thread_num; ++i) { + threads.at(i).join(); + } + printf("time elapsed during query process: %lfs\n", double(time(NULL) - start_time)); + + // 4. execute query and verify result + htable_filter.clear_columns(); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + key1.set_varbinary(ObString::make_string(rowkey)); + key2.set_varbinary(ObString::make_string(qualifier)); + int64_t read_int = 0; + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ObString str; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + ASSERT_EQ(OB_SUCCESS, ObHTableUtils::java_bytes_to_int64(str, read_int)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + // ASSERT_EQ(key3, ts); + std::cout << "key3: " << ts.get_int() << std::endl; + ASSERT_EQ(read_int, thread_num * inc_times); + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} + TEST_F(TestBatchExecute, htable_append) { // setup @@ -11772,7 +11879,6 @@ TEST_F(TestBatchExecute, atomic_batch_ops) service_client_->free_table(table); table = NULL; } - // create table if not exists auto_increment_defensive_test // (C1 bigint AUTO_INCREMENT primary key) PARTITION BY KEY(C1) PARTITIONS 16; TEST_F(TestBatchExecute, auto_increment_auto_increment_defensive) @@ -11794,6 +11900,273 @@ TEST_F(TestBatchExecute, auto_increment_auto_increment_defensive) ObTableOperationResult r; ASSERT_EQ(OB_NOT_SUPPORTED, the_table->execute(table_operation, r)); } +// create table if not exists htable1_cf1_check_and_delete(K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substr(K,1,32)) STORED, primary key(K, Q, T));" $db +TEST_F(TestBatchExecute, htable_check_and_put) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_check_and_put"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const char* rowkey = "row1"; + const char *qualifier = "cq1"; + // set query of checkAndPut + ObTableQueryAndMutate query_and_mutate; + ObTableQuery &query = query_and_mutate.get_query(); + ObTableBatchOperation &mutations = query_and_mutate.get_mutations(); + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); + ObHTableFilter &htable_filter = query.htable_filter(); + ObObj key1, key2, key3, value; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + // check with value is null three times + // put1: row1, cq1, "", check success + // put2: row1, cq1, "string100", check success + // put3: row1, cq1, "string101", check failed + htable_filter.add_column(ObString::make_string(qualifier)); + htable_filter.set_filter(ObString::make_string("CheckAndMutateFilter(=, 'binary:', 'cf1', 'cq1', true)")); + htable_filter.set_valid(true); + const char *values[3] = {"", "string100", "string200"}; + for (int i = 0; i < 3; i++) { + mutations.reset(); + entity->reset(); + key1.set_varbinary(ObString::make_string(rowkey)); + key2.set_varbinary(ObString::make_string(qualifier)); + key3.set_int(INT64_MAX); + value.set_varbinary(ObString::make_string(values[i])); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, mutations.insert_or_update(*entity)); + ObTableQueryAndMutateResult result; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query_and_mutate(query_and_mutate, result)); + if (i < 2) { + ASSERT_EQ(1, result.affected_rows_); + } else { + ASSERT_EQ(0, result.affected_rows_); + } + } + // execute query and verify result + htable_filter.reset(); + htable_filter.add_column(ObString::make_string(qualifier)); + htable_filter.set_max_versions(INT32_MAX); // get all versions + htable_filter.set_valid(true); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ObObj rk, cq, ts, val; + ObString str; + for (int j = 0; j < 2; j++) { + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_TRUE(str.compare(values[1-j]) == 0); + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} +TEST_F(TestBatchExecute, htable_check_and_put_multi_thread) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_check_and_put"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const char* rowkey = "row2"; + const char *qualifier = "cq2"; + ObString val_str = "string0"; + // set query of checkAndPut + ObTableQueryAndMutate query_and_mutate; + ObTableQuery &query = query_and_mutate.get_query(); + ObTableBatchOperation &mutations = query_and_mutate.get_mutations(); + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); + ObHTableFilter &htable_filter = query.htable_filter(); + htable_filter.add_column(ObString::make_string(qualifier)); + htable_filter.set_filter(ObString::make_string("CheckAndMutateFilter(=, 'substring:string0', 'cf1', 'cq2', true)")); + htable_filter.set_valid(true); + // set put mutation + mutations.reset(); + ObObj key1, key2, key3, value; + key1.set_varbinary(ObString::make_string(rowkey)); + key2.set_varbinary(ObString::make_string(qualifier)); + key3.set_int(INT64_MAX); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + value.set_varbinary(val_str); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, mutations.insert_or_update(*entity)); + int64_t total_affected_rows = 0; + // 3. execute checkAndPut concurently with value is null + auto task = [&](ObTable* one_table, const ObTableQueryAndMutate &query_and_mutate) { + ObTableQueryAndMutateResult inc_result; + ASSERT_EQ(OB_SUCCESS, one_table->execute_query_and_mutate(query_and_mutate, inc_result)); + (void)ATOMIC_AAF(&total_affected_rows, inc_result.affected_rows_); + }; + constexpr uint64_t thread_num = 100; + std::vector threads; + time_t start_time = time(NULL); + printf("begin to run tasks, thread_num: %ld\n", thread_num); + for (uint64_t i = 0; i < thread_num; ++i) { + std::thread t(task, the_table, query_and_mutate); + threads.push_back(std::move(t)); + } + for (uint64_t i = 0; i < thread_num; ++i) { + threads.at(i).join(); + } + printf("time elapsed during checkAndPut process: %lfs\n", double(time(NULL) - start_time)); + + // 4. execute query and verify result + htable_filter.reset(); + htable_filter.add_column(ObString::make_string(qualifier)); + htable_filter.set_max_versions(INT32_MAX); // get all versions + htable_filter.set_valid(true); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ObString str; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_EQ(str, val_str); + ASSERT_EQ(1, ATOMIC_LOAD(&total_affected_rows)); + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} + +// execute multiply checkAndPut(with check null and put empty) and one put(with value is not empty) operation +// concurrently, the newest cell value should generated by the put opreation +TEST_F(TestBatchExecute, htable_check_and_put_put) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_check_and_put_put"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const char* rowkey = "row2"; + const char *qualifier = "cq2"; + const char* null_val = ""; + // set query of checkAndPut + ObTableQueryAndMutate query_and_mutate; + ObTableQuery &query = query_and_mutate.get_query(); + ObTableBatchOperation &mutations = query_and_mutate.get_mutations(); + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); + ObHTableFilter &htable_filter = query.htable_filter(); + htable_filter.add_column(ObString::make_string(qualifier)); + htable_filter.set_filter(ObString::make_string("CheckAndMutateFilter(=, 'substring:string0', 'cf1', 'cq2', true)")); + htable_filter.set_valid(true); + // set put mutation + mutations.reset(); + ObObj key1, key2, key3, value; + key1.set_varbinary(ObString::make_string(rowkey)); + key2.set_varbinary(ObString::make_string(qualifier)); + key3.set_int(INT64_MAX); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + value.set_varbinary(null_val); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, mutations.insert_or_update(*entity)); + // set htable put operation with value is not null + ObTableBatchOperation put_op; + ObString put_val_str = "hello world"; + ObObj put_value; + ObITableEntity *put_entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != put_entity); + ASSERT_EQ(OB_SUCCESS, put_entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, put_entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, put_entity->add_rowkey_value(key3)); + put_value.set_varbinary(put_val_str); + ASSERT_EQ(OB_SUCCESS, put_entity->set_property(V, put_value)); + ASSERT_EQ(OB_SUCCESS, put_op.insert_or_update(*put_entity)); + // 3. execute checkAndPut concurently with check value is null and put value is null + auto check_and_put_task = [&](ObTable* one_table, const ObTableQueryAndMutate &query_and_mutate) { + ObTableQueryAndMutateResult inc_result; + int one_ret = one_table->execute_query_and_mutate(query_and_mutate, inc_result); + ASSERT_TRUE(one_ret == OB_SUCCESS || one_ret == OB_TRY_LOCK_ROW_CONFLICT); + }; + auto put_task = [&]() { + ObTableBatchOperationResult put_result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(put_op, put_result)); + ASSERT_EQ(1, put_result.count()); + }; + std::thread put_t(put_task); + constexpr uint64_t thread_num = 200; + std::vector threads; + time_t start_time = time(NULL); + printf("begin to run tasks, thread_num: %ld\n", thread_num); + for (uint64_t i = 0; i < thread_num; ++i) { + std::thread t(check_and_put_task, the_table, query_and_mutate); + threads.push_back(std::move(t)); + } + put_t.join(); + for (uint64_t i = 0; i < thread_num; ++i) { + threads.at(i).join(); + } + printf("time elapsed during checkAndPut process: %lfs\n", double(time(NULL) - start_time)); + + // 4. execute query and verify result + htable_filter.reset(); + htable_filter.add_column(ObString::make_string(qualifier)); + htable_filter.set_max_versions(1); // get all versions + htable_filter.set_valid(true); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ObString str; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_EQ(str, put_val_str); + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} int main(int argc, char **argv) { diff --git a/src/libtable/test/run_test_table_api.sh b/src/libtable/test/run_test_table_api.sh index e0e0e2d078..d3d5b5bea8 100755 --- a/src/libtable/test/run_test_table_api.sh +++ b/src/libtable/test/run_test_table_api.sh @@ -70,7 +70,7 @@ mysql -h $HOST -P $PORT -u $user -e "drop table if exists secondary_index_test; # hbase api mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1; create table if not exists htable1_cf1 (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substr(K,1,32)) STORED, primary key(K, Q, T));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_reverse; create table if not exists htable1_cf1_reverse like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_ttl; create table if not exists htable1_cf1_ttl (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) comment='{\"HColumnDescriptor\": {\"TimeToLive\": 5}}'" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_ttl; create table if not exists htable1_cf1_ttl (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) kv_attributes='{\"Hbase\": {\"TimeToLive\": 5}}'" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_filter; create table if not exists htable1_cf1_filter like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_delete; create table if not exists htable1_cf1_delete like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_put; create table if not exists htable1_cf1_put like htable1_cf1" $db @@ -81,6 +81,8 @@ mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment_ mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_append; create table if not exists htable1_cf1_append like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_empty_cq; create table if not exists htable1_cf1_empty_cq (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_query_sync; create table if not exists htable1_query_sync (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_check_and_put; create table if not exists htable1_cf1_check_and_put like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_check_and_put_put; create table if not exists htable1_cf1_check_and_put_put like htable1_cf1" $db # run ./test_table_api "$HOST" "$PORT" "$tenant_name" "$user_name" "$passwd" "$db" "$table_name" $RPCPORT #--gtest_filter=TestBatchExecute. @@ -138,9 +140,11 @@ mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment; mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment_empty; create table if not exists htable1_cf1_increment_empty like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_append; create table if not exists htable1_cf1_append like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_reverse; create table if not exists htable1_cf1_reverse like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_ttl; create table if not exists htable1_cf1_ttl (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) comment='{\"HColumnDescriptor\": {\"TimeToLive\": 5}}'" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_ttl; create table if not exists htable1_cf1_ttl (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) kv_attributes='{\"Hbase\": {\"TimeToLive\": 5}}'" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_empty_cq; create table if not exists htable1_cf1_empty_cq (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_query_sync; create table if not exists htable1_query_sync (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_check_and_put; create table if not exists htable1_cf1_check_and_put like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_check_and_put_put; create table if not exists htable1_cf1_check_and_put_put like htable1_cf1" $db # run ./test_table_api "$HOST" "$PORT" "$tenant_name" "$user_name" "$passwd" "$db" "$table_name" $RPCPORT diff --git a/src/logservice/ob_log_base_type.h b/src/logservice/ob_log_base_type.h index 32f06735fa..778c72e840 100644 --- a/src/logservice/ob_log_base_type.h +++ b/src/logservice/ob_log_base_type.h @@ -122,9 +122,9 @@ enum ObLogBaseType // only use role change service, do not write clog LS_BLOCK_TX_SERVICE_LOG_BASE_TYPE = 37, - // for workload repository service WORKLOAD_REPOSITORY_SERVICE_LOG_BASE_TYPE = 38, + TTL_LOG_BASE_TYPE = 39, // pay attention!!! // add log type in log_base_type_to_string // max value @@ -217,6 +217,8 @@ int log_base_type_to_string(const ObLogBaseType log_type, strncpy(str ,"BLOCK_TX_SERVICE", str_len); } else if (log_type == WORKLOAD_REPOSITORY_SERVICE_LOG_BASE_TYPE) { strncpy(str ,"WORKLOAD_REPOSITORY_SERVICE", str_len); + } else if (log_type == TTL_LOG_BASE_TYPE) { + strncpy(str ,"TTL_SERVICE", str_len); } else { ret = OB_INVALID_ARGUMENT; } diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 9a1bc0e88c..9fdf6be737 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2258,6 +2258,11 @@ typedef enum ObItemType T_ALTER_SYSTEM_KILL, // used to support kill session in oracle T_LOB_STORAGE_CLAUSE, + T_TABLE_TTL, + T_TTL_DEFINITION, + T_TTL_EXPR, + T_REMOVE_TTL, + T_KV_ATTRIBUTES, T_VALUES_TABLE_EXPRESSION,//used to values statement T_VALUES_ROW_LIST,//used to values statement T_MAX //Attention: add a new type before T_MAX diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 30ffe7522e..87a2a0857c 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -137,6 +137,7 @@ ob_set_subtarget(ob_server table table/ob_table_rpc_processor.cpp table/ob_table_query_sync_processor.cpp table/ob_table_service.cpp + table/ob_htable_lock_mgr.cpp table/ob_table_context.cpp table/ob_table_executor.cpp table/ob_table_delete_executor.cpp @@ -155,6 +156,11 @@ ob_set_subtarget(ob_server table table/ob_table_query_common.cpp table/ob_table_direct_load_processor.cpp table/ob_table_aggregation.cpp + table/ttl/ob_ttl_service.cpp + table/ttl/ob_tenant_ttl_manager.cpp + table/ttl/ob_tenant_tablet_ttl_mgr.cpp + table/ttl/ob_table_ttl_task.cpp + table/ttl/ob_table_ttl_executor.cpp ) ob_set_subtarget(ob_server table_load diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 0d02e68152..cac22c7b3c 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -72,6 +72,7 @@ #include "rootserver/ob_primary_ls_service.h" // for ObPrimaryLSService #include "sql/session/ob_sql_session_info.h" #include "sql/session/ob_sess_info_verify.h" +#include "observer/table/ttl/ob_ttl_service.h" namespace oceanbase { @@ -2805,6 +2806,30 @@ int ObRpcGetLSReplayedScnP::process() return ret; } +int ObTenantTTLP::process() +{ + int ret = OB_SUCCESS; + ObTTLRequestArg &req = arg_; + ObTTLResponseArg &res = result_; + table::ObTTLService *ttl_service = nullptr; + + if (OB_UNLIKELY(!req.is_valid())) { + ret = OB_INVALID_ARGUMENT; + RS_LOG(WARN, "invalid argument", K(ret), K(req)); + } else if (OB_UNLIKELY(req.tenant_id_ != MTL_ID())) { + ret = OB_ERR_UNEXPECTED; + RS_LOG(ERROR, "mtl_id not match", K(ret), K(req), "mtl_id", MTL_ID()); + } else if (OB_ISNULL(ttl_service = MTL(table::ObTTLService*))) { + ret = OB_ERR_UNEXPECTED; + RS_LOG(ERROR, "ttl service is nullptr", KR(ret), K(req)); + } else if (OB_FAIL(ttl_service->launch_ttl_task(req))) { + RS_LOG(WARN, "fail to launch ttl", KR(ret), K(req)); + } + res.err_code_ = ret; + ret = OB_SUCCESS; + return ret; +} + int ObAdminUnlockMemberListP::process() { int ret = OB_SUCCESS; diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index 38f0d9d18f..166ab217aa 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -253,6 +253,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_GET_SERVER_RESOURCE_INFO, ObRpcGetServerResourceIn OB_DEFINE_PROCESSOR_S(Srv, OB_UPDATE_TENANT_INFO_CACHE, ObUpdateTenantInfoCacheP); OB_DEFINE_PROCESSOR_S(Srv, OB_BROADCAST_CONSENSUS_VERSION, ObBroadcastConsensusVersionP); OB_DEFINE_PROCESSOR_S(Srv, OB_GET_LS_REPLAYED_SCN, ObRpcGetLSReplayedScnP); +OB_DEFINE_PROCESSOR_S(Srv, OB_TABLE_TTL, ObTenantTTLP); OB_DEFINE_PROCESSOR_S(Srv, OB_HA_UNLOCK_MEMBER_LIST, ObAdminUnlockMemberListP); } // end of namespace observer diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 58c4384c45..ddb5b269de 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -106,6 +106,7 @@ #include "observer/virtual_table/ob_mds_event_buffer.h" #include "observer/ob_server_startup_task_handler.h" #include "share/detect/ob_detect_manager.h" +#include "observer/table/ttl/ob_table_ttl_task.h" #ifdef OB_BUILD_ARBITRATION #include "logservice/arbserver/palf_env_lite_mgr.h" #include "logservice/arbserver/ob_arb_srv_network_frame.h" diff --git a/src/observer/ob_srv_xlator_partition.cpp b/src/observer/ob_srv_xlator_partition.cpp index b53e4d9735..1e69e96102 100644 --- a/src/observer/ob_srv_xlator_partition.cpp +++ b/src/observer/ob_srv_xlator_partition.cpp @@ -199,6 +199,7 @@ void oceanbase::observer::init_srv_xlator_for_others(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObTableQueryAndMutateP, gctx_); RPC_PROCESSOR(ObTableQuerySyncP, gctx_); RPC_PROCESSOR(ObTableDirectLoadP, gctx_); + RPC_PROCESSOR(ObTenantTTLP, gctx_); // HA GTS RPC_PROCESSOR(ObHaGtsPingRequestP, gctx_); diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 5d53d2bc04..e90f600d0c 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -131,10 +131,12 @@ #include "rootserver/ob_rs_event_history_table_operator.h" #include "rootserver/ob_heartbeat_service.h" #include "share/detect/ob_detect_manager.h" +#include "observer/table/ttl/ob_ttl_service.h" #ifdef ERRSIM #include "share/errsim_module/ob_tenant_errsim_module_mgr.h" #include "share/errsim_module/ob_tenant_errsim_event_mgr.h" #endif +#include "observer/table/ob_htable_lock_mgr.h" using namespace oceanbase; using namespace oceanbase::lib; @@ -518,12 +520,14 @@ int ObMultiTenant::init(ObAddr myaddr, // mtl_wait_default, mtl_destroy_default); } MTL_BIND2(mtl_new_default, rootserver::ObHeartbeatService::mtl_init, nullptr, rootserver::ObHeartbeatService::mtl_stop, rootserver::ObHeartbeatService::mtl_wait, mtl_destroy_default); + MTL_BIND2(mtl_new_default, table::ObTTLService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); #ifdef ERRSIM MTL_BIND2(mtl_new_default, ObTenantErrsimModuleMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantErrsimEventMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); #endif + MTL_BIND(table::ObHTableLockMgr::mtl_init, table::ObHTableLockMgr::mtl_destroy); MTL_BIND2(mtl_new_default, ObSharedTimer::mtl_init, ObSharedTimer::mtl_start, ObSharedTimer::mtl_stop, ObSharedTimer::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObOptStatMonitorManager::mtl_init, ObOptStatMonitorManager::mtl_start, ObOptStatMonitorManager::mtl_stop, ObOptStatMonitorManager::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantSrs::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); diff --git a/src/observer/table/ob_htable_filter_operator.cpp b/src/observer/table/ob_htable_filter_operator.cpp index f5ecbc2c30..874fe915d3 100644 --- a/src/observer/table/ob_htable_filter_operator.cpp +++ b/src/observer/table/ob_htable_filter_operator.cpp @@ -15,46 +15,31 @@ #include "ob_htable_utils.h" #include "lib/json/ob_json.h" #include "share/ob_errno.h" +#include "share/table/ob_ttl_util.h" using namespace oceanbase::common; using namespace oceanbase::table; using namespace oceanbase::table::hfilter; +using namespace oceanbase::share::schema; -int ObHColumnDescriptor::from_string(const common::ObString &str) +// format: {"Hbase": {"TimeToLive": 3600, "MaxVersions": 3}} +int ObHColumnDescriptor::from_string(const common::ObString &kv_attributes) { + reset(); int ret = OB_SUCCESS; - ObArenaAllocator allocator; - json::Parser json_parser; - json::Value *ast = NULL; - if (str.empty()) { - // skip - } else if (OB_FAIL(json_parser.init(&allocator))) { - LOG_WARN("failed to init json parser", K(ret)); - } else if (OB_FAIL(json_parser.parse(str.ptr(), str.length(), ast))) { - LOG_WARN("failed to parse", K(ret), K(str)); - ret = OB_SUCCESS; - } else if (NULL != ast - && ast->get_type() == json::JT_OBJECT - && ast->get_object().get_size() == 1) { - json::Pair *kv = ast->get_object().get_first(); - if (NULL != kv && kv != ast->get_object().get_header()) { - if (kv->name_.case_compare("HColumnDescriptor") == 0) { - ast = kv->value_; - if (NULL != ast && ast->get_type() == json::JT_OBJECT) { - DLIST_FOREACH(elem, ast->get_object()) { - if (elem->name_.case_compare("TimeToLive") == 0) { - json::Value *ttl_val = elem->value_; - if (NULL != ttl_val && ttl_val->get_type() == json::JT_NUMBER) { - time_to_live_ = static_cast(ttl_val->get_number()); - } - } - } // end foreach - } - } - } + if (kv_attributes.empty()) { + // do nothing + } else if (OB_FAIL(ObTTLUtil::parse_kv_attributes(kv_attributes, max_version_, time_to_live_))) { + LOG_WARN("fail to parse kv attributes", K(ret), K(kv_attributes)); } return ret; } +void ObHColumnDescriptor::reset() +{ + time_to_live_ = 0; + max_version_ = 0; +} + //////////////////////////////////////////////////////////////// class ObHTableColumnTracker::ColumnCountComparator { @@ -82,6 +67,19 @@ void ObHTableColumnTracker::set_ttl(int32_t ttl_value) oldest_stamp_ = now - (ttl_value * 1000LL); LOG_DEBUG("[yzfdebug] set ttl", K(ttl_value), K(now), K_(oldest_stamp)); NG_TRACE_EXT(t, OB_ID(arg1), ttl_value, OB_ID(arg2), oldest_stamp_); + } else { + LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid ttl value", K(ttl_value)); + } +} + +void ObHTableColumnTracker::set_max_version(int32_t max_version) +{ + if (max_version > 0) { + max_versions_ = max_version; + LOG_DEBUG("set max_version", K(max_version)); + NG_TRACE_EXT(version, OB_ID(arg1), max_version); + } else { + LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid max version value", K(max_version)); } } @@ -750,8 +748,14 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) } if (OB_FAIL(column_tracker_->init(htable_filter_, scan_order_))) { LOG_WARN("failed to init column tracker", K(ret)); - } else if (time_to_live_ > 0) { - column_tracker_->set_ttl(time_to_live_); + } else { + if (time_to_live_ > 0) { + column_tracker_->set_ttl(time_to_live_); + } + if (max_version_ > 0) { + int32_t real_max_version = std::min(column_tracker_->get_max_version(), max_version_); + column_tracker_->set_max_version(real_max_version); + } } } if (OB_SUCC(ret) && NULL == matcher_) { diff --git a/src/observer/table/ob_htable_filter_operator.h b/src/observer/table/ob_htable_filter_operator.h index bf500966b3..413615535a 100644 --- a/src/observer/table/ob_htable_filter_operator.h +++ b/src/observer/table/ob_htable_filter_operator.h @@ -22,6 +22,7 @@ #include "ob_htable_filters.h" #include "ob_table_scan_executor.h" #include +#include "share/schema/ob_table_schema.h" namespace oceanbase { @@ -31,14 +32,19 @@ class ObHColumnDescriptor final { public: ObHColumnDescriptor() - :time_to_live_(0) + :time_to_live_(0), + max_version_(0) {} - int from_string(const common::ObString &str); + void reset(); + int from_string(const common::ObString &kv_attributes); void set_time_to_live(int32_t v) { time_to_live_ = v; } int32_t get_time_to_live() const { return time_to_live_; } + void set_max_version(int32_t v) { max_version_ = v; } + int32_t get_max_version() const { return max_version_; } private: int32_t time_to_live_; // Time-to-live of cell contents, in seconds. + int32_t max_version_; }; enum class ObHTableMatchCode @@ -86,6 +92,8 @@ public: // Give the tracker a chance to declare it's done based on only the timestamp. bool is_done(int64_t timestamp) const; void set_ttl(int32_t ttl_value); + void set_max_version(int32_t max_version); + int32_t get_max_version() { return max_versions_; } protected: int32_t max_versions_; // default: 1 int32_t min_versions_; // default: 0 @@ -215,6 +223,7 @@ public: int add_same_kq_to_res(ObIArray &same_kq_cells, ObTableQueryResult *&out_result); ObIArray &get_same_kq_cells() { return same_kq_cells_; } + void set_max_version(int32_t max_version) { max_version_ = max_version; } private: int next_cell(); int reverse_next_cell(ObIArray &same_kq_cells, @@ -231,7 +240,8 @@ private: int32_t offset_per_row_per_cf_; int64_t max_result_size_; int32_t batch_size_; - int32_t time_to_live_; // Time-to-live of cell contents, in seconds. + int32_t time_to_live_; // Column family level time-to-live, in seconds. + int32_t max_version_; // Column family max_version table::ObTableQueryResult one_hbase_row_; ObHTableCellEntity curr_cell_; @@ -265,8 +275,10 @@ public: row_iterator_.set_scan_result(scan_result); } void set_ttl(int32_t ttl_value) { row_iterator_.set_ttl(ttl_value); } + void set_max_version(int32_t max_version_value) { row_iterator_.set_max_version(max_version_value); } // parse the filter string int parse_filter_string(common::ObIAllocator* allocator); + OB_INLINE table::hfilter::Filter *get_hfiter() { return hfilter_; } private: ObHTableRowIterator row_iterator_; table::ObTableQueryResult *one_result_; diff --git a/src/observer/table/ob_htable_filters.cpp b/src/observer/table/ob_htable_filters.cpp index 7751c239f6..651e0a4769 100644 --- a/src/observer/table/ob_htable_filters.cpp +++ b/src/observer/table/ob_htable_filters.cpp @@ -860,6 +860,12 @@ void CheckAndMutateFilter::reset() matched_column_ = false; } +// NOTE: when value_is_null is true，cannot return other cell directly +// 判断是否 check 通过，例如当整个 column family 为空的时候，result_count = 0，但是应该是 check 通过 + +// NOTE: when value_is_null is true, cannot return other cells directly and determine whether check is +// passed by result_count > 0 in ObTableQueryAndMutateP::try_process. +// for example, if the whole column family is empty and result_count = 0, but check should passed. int CheckAndMutateFilter::filter_cell(const ObHTableCell &cell, ReturnCode &ret_code) { int ret = OB_SUCCESS; @@ -869,29 +875,18 @@ int CheckAndMutateFilter::filter_cell(const ObHTableCell &cell, ReturnCode &ret_ LOG_DEBUG("[yzfdebug] already matched column", K(ret_code)); } else if (found_column_) { // latest_version_only_ == true // found but not matched the column - if (value_is_null_) { - ret_code = ReturnCode::INCLUDE; - } else { - ret_code = ReturnCode::NEXT_ROW; - } + ret_code = ReturnCode::NEXT_ROW; LOG_DEBUG("[yzfdebug] latest verion only but not matched", K(ret_code)); - } else if (!match_column(cell)) { - ret_code = ReturnCode::INCLUDE; - LOG_DEBUG("[yzfdebug] not found column yet", K(ret_code)); - } else { + } else if (match_column(cell)) { found_column_ = true; LOG_DEBUG("[yzfdebug] found column", K_(found_column)); - if (value_is_null_) { - ret_code = ReturnCode::NEXT_ROW; + if (value_is_null_ || !filter_column_value(cell)) { + matched_column_ = true; + ret_code = ReturnCode::INCLUDE; + LOG_DEBUG("[yzfdebug] found column and match", K(ret_code)); } else { - if (filter_column_value(cell)) { - ret_code = ReturnCode::NEXT_ROW; - LOG_DEBUG("[yzfdebug] found column but value not match", K(ret_code)); - } else { - matched_column_ = true; - ret_code = ReturnCode::INCLUDE; - LOG_DEBUG("[yzfdebug] found column and match", K(ret_code)); - } + ret_code = ReturnCode::NEXT_ROW; + LOG_DEBUG("[yzfdebug] found column but value not match", K(ret_code)); } } return ret; @@ -912,10 +907,6 @@ bool CheckAndMutateFilter::filter_row() { LOG_DEBUG("[yzfdebug] filter row", K_(found_column), K_(matched_column), K_(value_is_null)); bool bret = true; - if (value_is_null_) { - bret = found_column_; - } else { - bret = found_column_ ? (!matched_column_) : (false/*filter_if_missing*/); - } + bret = found_column_ ? (!matched_column_) : true; return bret; } diff --git a/src/observer/table/ob_htable_filters.h b/src/observer/table/ob_htable_filters.h index fed13408b8..5f7eb0fd6b 100644 --- a/src/observer/table/ob_htable_filters.h +++ b/src/observer/table/ob_htable_filters.h @@ -491,6 +491,7 @@ public: virtual int filter_cell(const ObHTableCell &cell, ReturnCode &ret_code) override; virtual bool filter_row() override; virtual bool has_filter_row() override { return true; } + OB_INLINE bool value_is_null() { return value_is_null_; } TO_STRING_KV("filter", "CheckAndMutateFilter", K_(family), K_(qualifier), diff --git a/src/observer/table/ob_htable_lock_mgr.cpp b/src/observer/table/ob_htable_lock_mgr.cpp new file mode 100644 index 0000000000..7340d7101e --- /dev/null +++ b/src/observer/table/ob_htable_lock_mgr.cpp @@ -0,0 +1,458 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_htable_lock_mgr.h" +using namespace oceanbase::common; + +namespace oceanbase +{ +namespace table +{ + +const char *OB_HTABLE_LOCK_MANAGER = "hTableLockMgr"; + +int ObHTableLockMgr::mtl_init(ObHTableLockMgr *&htable_lock_mgr) { + int ret = OB_SUCCESS; + htable_lock_mgr = OB_NEW(ObHTableLockMgr, ObMemAttr(MTL_ID(), OB_HTABLE_LOCK_MANAGER)); + if (OB_ISNULL(htable_lock_mgr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for ObHTableLockMgr", K(ret)); + } else if (OB_FAIL(htable_lock_mgr->init())) { + LOG_WARN("failed to init htable lock manager", K(ret)); + } + return ret; +} + +int ObHTableLockMgr::init() +{ + int ret = OB_SUCCESS; + const ObMemAttr attr(MTL_ID(), OB_HTABLE_LOCK_MANAGER); + SpinWLockGuard guard(spin_lock_); + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_FAIL(lock_map_.create(DEFAULT_BUCKET_NUM, ObModIds::TABLE_PROC, ObModIds::TABLE_PROC, MTL_ID()))) { + LOG_WARN("fail to create htable lock map", K(ret)); + } else if (allocator_.init(ObMallocAllocator::get_instance(), OB_MALLOC_MIDDLE_BLOCK_SIZE, attr)) { + LOG_WARN("fail to init allocator", K(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +void ObHTableLockMgr::mtl_destroy(ObHTableLockMgr *&htable_lock_mgr) +{ + if (nullptr != htable_lock_mgr) { + LOG_INFO("trace ObHTableLockMgr destroy", K(MTL_ID())); + htable_lock_mgr->lock_map_.destroy(); + common::ob_delete(htable_lock_mgr); + htable_lock_mgr = nullptr; + } +} + +// acquire_handle without tx_id must set_tx_id later +int ObHTableLockMgr::acquire_handle(ObHTableLockHandle *&handle) +{ + int ret = OB_SUCCESS; + void *buf = allocator_.alloc(sizeof(ObHTableLockHandle)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(sizeof(ObHTableLockHandle))); + } else { + handle = new(buf) ObHTableLockHandle(); + } + return ret; +} + +int ObHTableLockMgr::acquire_handle(const transaction::ObTransID &tx_id, ObHTableLockHandle *&handle) +{ + int ret = OB_SUCCESS; + void *buf = allocator_.alloc(sizeof(ObHTableLockHandle)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(sizeof(ObHTableLockHandle))); + } else { + handle = new(buf) ObHTableLockHandle(tx_id); + } + return ret; +} + +int ObHTableLockHandle::find_lock_node(uint64_t table_id, const common::ObString &key, ObHTableLockNode *&lock_node) +{ + int ret = OB_SUCCESS; + ObHTableLockNode *cur_node = lock_nodes_; + lock_node = nullptr; + ObHTableLockKey target_lock_key(table_id, key); + bool is_matched = false; + while (OB_SUCC(ret) && cur_node != nullptr && !is_matched) { + ObHTableLockKey *lock_key = cur_node->get_lock_key(); + if (OB_ISNULL(lock_key)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null lock key", K(ret)); + } else if (!lock_key->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid lock key", K(ret), KPC(lock_key)); + } else if (*lock_key == target_lock_key) { + is_matched = true; + lock_node = cur_node; + } else {/* do nothing */} + cur_node = cur_node->next_; + } + return ret; +} + +// situations to consider when locking +// 1. add shared lock +// 2. add exclusive lock +// 3. add shared lock repeatly: do nothing +// 4. add exclusive lock repeatly: do nothing +// 5. escalate shared lock to exclusive lock: only allowed when no one else has the same read lock +// 6. add shared lock when holding exclusive lock: do nothing +int ObHTableLockMgr::lock_row(const uint64_t table_id, const common::ObString& key, ObHTableLockMode mode, ObHTableLockHandle &handle) +{ + int ret = OB_SUCCESS; + ObHTableLockNode *lock_node = nullptr; + if (key.empty()) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("null lock row key", K(ret)); + } else if (handle.find_lock_node(table_id, key, lock_node)) { + LOG_WARN("fail to find lock node", K(ret), K(table_id), K(key)); + } else if (OB_ISNULL(lock_node)) { + if (OB_FAIL(internal_lock_row(table_id, key, mode, handle))) { + LOG_WARN("fail to lock", K(ret), K(table_id), K(key), K(mode)); + } else {/* do nothing */} + } else if (lock_node->lock_mode_ == SHARED && mode == EXCLUSIVE) { + if (OB_FAIL(rd2wrlock(*lock_node))) { + LOG_WARN("fail to escalate read lock to write lock", K(ret)); + } else { + lock_node->lock_mode_ = EXCLUSIVE; + } + } else {/* do nothing */} + + return ret; +} + +int ObHTableLockMgr::internal_lock_row(const uint64_t table_id, const common::ObString& key, ObHTableLockMode lock_mode, ObHTableLockHandle &handle) +{ + int ret = OB_SUCCESS; + ObHTableLockKey tmp_lock_key(table_id, key); + ObHTableLockOp lock_op(lock_mode); + ObHTableLockNode *lock_node = nullptr; + ObHTableLockKey *new_lock_key = nullptr; + ObHTableLock *new_lock = nullptr; + bool lock_exists = false; + + if (OB_FAIL(alloc_lock_node(lock_mode, lock_node))) { + LOG_WARN("fail to alloc lock node", K(ret), K(lock_mode)); + } else { + // suppose the lock exists firstly, try to get and add shared/exclusive lock + if (OB_FAIL(lock_map_.atomic_refactored(&tmp_lock_key, lock_op))) { + if (ret == OB_HASH_NOT_EXIST) { + // lock not exists, try to add a new lock or use the lock added by others + ret = OB_SUCCESS; + if (OB_FAIL(alloc_lock_key(table_id, key, new_lock_key))) { + LOG_WARN("fail to alloc new lock key", K(ret)); + } else if (OB_FAIL(alloc_lock(lock_mode, new_lock))) { + LOG_WARN("fail to alloc new lock", K(ret)); + } else if (OB_FAIL(lock_map_.set_or_update(new_lock_key, new_lock, lock_op))) { + LOG_WARN("fail to set or update lock", K(ret)); + } + } else { + LOG_WARN("fail to add read lock", K(ret), K(tmp_lock_key)); + } + } + } + + if (OB_SUCC(ret) && lock_op.is_called()) { + ret = lock_op.get_ret(); + lock_exists = true; + } + + // add lock key to lock node, and add lock node to lock handle + if (OB_SUCC(ret)) { + if (lock_exists) { + ObHTableLockKey *old_lock_key = lock_op.get_lock_key(); + if (OB_ISNULL(old_lock_key)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("null old lock key but lock exist", K(ret)); + } else { + lock_node->set_lock_key(old_lock_key); + handle.add_lock_node(*lock_node); + } + } else { + lock_node->set_lock_key(new_lock_key); + handle.add_lock_node(*lock_node); + } + } + + if (OB_FAIL(ret) || lock_exists) { + if (OB_NOT_NULL(new_lock_key)) { + allocator_.free(new_lock_key->key_.ptr()); + allocator_.free(new_lock_key); + } + if (OB_NOT_NULL(new_lock)) { + allocator_.free(new_lock); + } + if (OB_FAIL(ret) && OB_NOT_NULL(lock_node)) { + allocator_.free(lock_node); + } + } + return ret; +} + + +int ObHTableLockMgr::alloc_lock_node(ObHTableLockMode lock_mode, ObHTableLockNode *&lock_node) +{ + int ret = OB_SUCCESS; + void *buf = allocator_.alloc(sizeof(ObHTableLockNode)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(sizeof(ObHTableLockNode))); + } else { + lock_node = new(buf) ObHTableLockNode(lock_mode); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(buf)) { + allocator_.free(buf); + } + return ret; +} + +int ObHTableLockMgr::alloc_lock_key(const uint64_t table_id, const ObString &key, ObHTableLockKey *&lock_key) +{ + int ret = OB_SUCCESS; + ObHTableLockKey *tmp_lock_key = nullptr; + void *buf = allocator_.alloc(sizeof(ObHTableLockKey)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(sizeof(ObHTableLockKey))); + } else { + tmp_lock_key = new(buf) ObHTableLockKey(); + if (OB_FAIL(ob_write_string(allocator_, key, tmp_lock_key->key_))) { + LOG_WARN("fail to copy key", K(ret)); + } else { + tmp_lock_key->table_id_ = table_id; + lock_key = tmp_lock_key; + } + } + + if (OB_FAIL(ret) && OB_NOT_NULL(buf)) { + allocator_.free(buf); + } + return ret; +} + +// alloc lock with initial lock mode +int ObHTableLockMgr::alloc_lock(ObHTableLockMode lock_mode, ObHTableLock *&lock) +{ + int ret = OB_SUCCESS; + ObHTableLock *tmp_lock = nullptr; + void *buf = allocator_.alloc(sizeof(ObHTableLock)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(sizeof(ObHTableLock))); + } else { + tmp_lock = new(buf) ObHTableLock(); + if (lock_mode == ObHTableLockMode::SHARED) { + if (OB_FAIL(tmp_lock->try_rdlock())) { + LOG_WARN("fail to add read lock", K(ret)); + } + } else if (lock_mode == ObHTableLockMode::EXCLUSIVE) { + if (OB_FAIL(tmp_lock->try_wrlock())) { + LOG_WARN("fail to add write lock", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid lock mode", K(ret), K(lock_mode)); + } + } + + if (OB_SUCC(ret)) { + lock = tmp_lock; + } else if (OB_NOT_NULL(buf)) { + allocator_.free(buf); + } else {/*do nothing*/} + + return ret; +} + +void ObHTableLockOp::operator() (common::hash::HashMapPair &entry) +{ + int ret = OB_SUCCESS; + is_called_ = true; + lock_success_ = false; + if (OB_ISNULL(entry.second)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null lock key", K(ret)); + } else { + if (lock_mode_ == ObHTableLockMode::SHARED) { + if (OB_FAIL(entry.second->try_rdlock())) { + LOG_WARN("fail to add read lock", K(ret)); + } + } else { + if (OB_FAIL(entry.second->try_wrlock())) { + LOG_WARN("fail to add write lock", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + old_lock_key_ = entry.first; + lock_success_ = true; + } + ret_code_ = ret; +} + +void ObHTableRd2WrLockOp::operator() (common::hash::HashMapPair &entry) +{ + int ret = OB_SUCCESS; + ObHTableLock *lock = entry.second; + if (OB_ISNULL(lock)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null lock key", K(ret)); + } else if (!lock->can_escalate_lock()) { + ret = OB_TRY_LOCK_ROW_CONFLICT; + LOG_WARN("can not escalate lock from share to exclusive", K(ret)); + } else if (OB_FAIL(lock->unlock())) { + LOG_ERROR("fail to unlock during escalate the lock", K(ret)); + } else if (OB_FAIL(lock->try_wrlock())) { + LOG_ERROR("fail to add write lock during escalate the lock", K(ret)); + } else {/* do nothing*/} + + ret_code_ = ret; +} + +bool ObHTableUnLockOpPred::operator() (common::hash::HashMapPair &entry) +{ + bool need_erase = false; + int ret = OB_SUCCESS; + ObHTableLock *lock = entry.second; + if (OB_ISNULL(lock)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null lock key", K(ret)); + } else if (OB_FAIL(lock->unlock())) { + LOG_ERROR("fail to unlock", K(ret)); + } else if (!lock->is_locked()) { + need_erase = true; + } + ret_code_ = ret; + return need_erase; +} + +// head insert new lock node into lock handle +void ObHTableLockHandle::add_lock_node(ObHTableLockNode &lock_node) +{ + ObHTableLockNode *orgin_lock_nodes = lock_nodes_; + lock_nodes_ = &lock_node; + lock_nodes_->next_ = orgin_lock_nodes; +} + +int ObHTableLockMgr::rd2wrlock(ObHTableLockNode &lock_node) +{ + int ret = OB_SUCCESS; + if (lock_node.get_lock_mode() != SHARED) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else { + ObHTableRd2WrLockOp rd2wrlockop; + if (OB_FAIL(lock_map_.atomic_refactored(lock_node.get_lock_key(), rd2wrlockop))) { + LOG_WARN("fail to escalate read lock to write lock", K(ret)); + } else { + ret = rd2wrlockop.get_ret(); + } + } + return ret; +} + +int ObHTableLockMgr::release_handle(ObHTableLockHandle &handle) +{ + int ret = OB_SUCCESS; + ObHTableLockNode *cur = handle.lock_nodes_; + ObHTableLockNode *next = nullptr; + while(cur != nullptr) { + next = cur->next_; + if (OB_FAIL(release_node(*cur))) { + LOG_ERROR("fail to release lock node", K(ret), KPC(cur)); + } + cur = next; + } + allocator_.free(&handle); + return ret; +} + +int ObHTableLockMgr::release_node(ObHTableLockNode &lock_node) +{ + int ret = OB_SUCCESS; + ObHTableUnLockOpPred unlock_op_pred; + ObHTableLockKey *lock_key = lock_node.lock_key_; + ObHTableLock *lock = nullptr; + bool is_erased = false; + if (OB_ISNULL(lock_key)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null lock key", K(ret)); + } else if (!lock_key->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid argument", K(ret), KPC(lock_key)); + } else if (OB_FAIL(lock_map_.erase_if(lock_key, unlock_op_pred, is_erased, &lock))) { + LOG_ERROR("fail to erase lock", K(ret), "unlock_op_pred.ret_code_", unlock_op_pred.get_ret()); + } else if (OB_FAIL(unlock_op_pred.get_ret())) { + LOG_ERROR("fail to unlock", K(ret)); + } else if (is_erased) { + allocator_.free(lock_key->key_.ptr()); + allocator_.free(lock_key); + if (OB_ISNULL(lock)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null lock", K(ret)); + } else { + allocator_.free(lock); + } + } else {/* do nothing */} + + allocator_.free(&lock_node); + return ret; +} + +bool ObHTableLockKey::operator==(const ObHTableLockKey &other) const +{ + return table_id_ == other.table_id_ && key_ == other.key_; +} + +int ObHTableLock::try_rdlock() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(latch_.try_rdlock(common::ObLatchIds::TABLE_API_LOCK))) { + // rewrite ret + ret = OB_TRY_LOCK_ROW_CONFLICT; + LOG_WARN("fail to try add read lock", K(ret)); + } + return ret; +} + +int ObHTableLock::try_wrlock() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(latch_.try_wrlock(common::ObLatchIds::TABLE_API_LOCK, NULL))) { + // rewrite ret + ret = OB_TRY_LOCK_ROW_CONFLICT; + LOG_WARN("fail to try add write lock", K(ret)); + } + return ret; +} + +int ObHTableLock::unlock() +{ + return latch_.unlock(NULL); +} + +} // end namespace table +} // end namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_htable_lock_mgr.h b/src/observer/table/ob_htable_lock_mgr.h new file mode 100644 index 0000000000..a28b87bd34 --- /dev/null +++ b/src/observer/table/ob_htable_lock_mgr.h @@ -0,0 +1,286 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBSERVER_TABLE_OB_HTABLE_LOCK_MGR_H +#define OBSERVER_TABLE_OB_HTABLE_LOCK_MGR_H +#include "lib/allocator/ob_fifo_allocator.h" +#include "lib/lock/ob_latch.h" +#include "storage/tx/ob_trans_define.h" + +namespace oceanbase +{ +namespace table +{ + +enum ObHTableLockMode +{ + SHARED, // for put, delete, + EXCLUSIVE // for check_and_mutate, increment, append +}; + +struct ObHTableLockKey +{ + ObHTableLockKey() : table_id_(common::OB_INVALID_ID), key_() {}; + ObHTableLockKey(uint64_t table_id, common::ObString key) : table_id_(table_id), key_(key) {} + ~ObHTableLockKey() + { + table_id_ = common::OB_INVALID_ID; + key_.reset(); + } + bool is_valid() { return table_id_ != common::OB_INVALID_ID && !key_.empty(); } + uint64_t hash() const + { + int64_t hash_value = key_.hash(); + hash_value = common::murmurhash(&table_id_, sizeof(table_id_), hash_value); + return hash_value; + } + inline int hash(uint64_t &hash_val) const + { + hash_val = hash(); + return OB_SUCCESS; + } + bool operator==(const ObHTableLockKey &other) const; + TO_STRING_KV(K_(table_id), K_(key)); + + uint64_t table_id_; + common::ObString key_; +}; + +// need to release all lock nodes it held before a lock handle is released +class ObHTableLockNode +{ + friend class ObHTableLockMgr; + friend class ObHTableLockHandle; +public: + explicit ObHTableLockNode(ObHTableLockMode lock_mode) + : lock_mode_(lock_mode), + lock_key_(nullptr), + next_(nullptr) + {} + ~ObHTableLockNode() {} + TO_STRING_KV(K_(lock_mode), KPC_(lock_key), KPC_(next)); + void set_lock_key(ObHTableLockKey *lock_key) { lock_key_ = lock_key; } + void set_lock_mode(ObHTableLockMode lock_mode) { lock_mode_ = lock_mode; } + ObHTableLockKey *get_lock_key() { return lock_key_; } + ObHTableLockMode get_lock_mode() { return lock_mode_; } + +private: + ObHTableLockMode lock_mode_; + ObHTableLockKey *lock_key_; + ObHTableLockNode *next_; +}; + +// records all htable locks held by a transaction +// after the transaction end, all the lock held by the lock handle should be released +class ObHTableLockHandle +{ + friend class ObHTableLockMgr; +public: + ObHTableLockHandle() : tx_id_(0), lock_nodes_(nullptr) {} + + ObHTableLockHandle(const transaction::ObTransID &tx_id) : tx_id_(tx_id), lock_nodes_(nullptr) {} + + ~ObHTableLockHandle() + { + tx_id_.reset(); + lock_nodes_ = nullptr; + } + void add_lock_node(ObHTableLockNode &lock_node); + int find_lock_node(uint64_t table_id, const common::ObString &key, ObHTableLockNode *&lock_node); + + void set_tx_id(transaction::ObTransID tx_id) { + tx_id_ = tx_id; + } + +private: + transaction::ObTransID tx_id_; + ObHTableLockNode *lock_nodes_; +}; + +class ObHTableLock +{ +public: + ObHTableLock() {} + ~ObHTableLock() {} + + /* try_rdlock - try to add read lock + * + * for read lock, ObHTableLock only record ref count, ObHTableHandle record all locks it holds + * + * return: + * OB_SUCCESS - OK + * OB_TRY_LOCK_ROW_CONFLICT - if lock conflict + */ + int try_rdlock(); + + /* try_wrlock - try to add write lock + * + * return: + * OB_SUCCESS - OK + * OB_TRY_LOCK_ROW_CONFLICT - if lock conflict + */ + int try_wrlock(); + + /* unlock: unlock htable read/write lock + * + * return: + * OB_SUCCESS - OK + * OB_ERR_UNEXPECTED - doesn't hold by anyone currently + */ + int unlock(); + + OB_INLINE bool can_escalate_lock() { return latch_.is_rdlocked() && latch_.get_rdcnt() == 1; } + OB_INLINE bool is_locked() { return latch_.is_locked(); } + OB_INLINE bool is_rdlocked() { return latch_.is_rdlocked(); } + OB_INLINE bool is_wrlocked() { return latch_.is_wrlocked(); } +private: + common::ObLatch latch_; +}; + +class ObHTableLockMgr +{ +public: + ObHTableLockMgr() : is_inited_(false), spin_lock_(), lock_map_(), allocator_(MTL_ID()) {} + ~ObHTableLockMgr() {} + /** + * acquire_handle - acquire htable lock handle + * + * this is the start of lifecycle of htable lock inside a transaction + * Notice: use acquire_handle without tx_id must set tx_id later! + * + * @handle: the htable lock handle returned + * + * Return: + * OB_SUCCESS - OK + */ + int acquire_handle(ObHTableLockHandle *&handle); + + /** + * acquire_handle - acquire htable lock handle + * + * this is the start of lifecycle of htable lock inside a transaction + * + * @handle: the htable lock handle returned + * + * Return: + * OB_SUCCESS - OK + */ + int acquire_handle(const transaction::ObTransID &tx_id, ObHTableLockHandle *&handle); + + /** + * lock_row - lock the row in htable model + * + * @table_id: the table(column family) where the row is to be locked + * @key: the target htable rowkey + * @mode: share or exclusive lock mode + * @handle: the target htable lock handle + * + * Return: + * OB_SUCCESS - OK + * OB_TRY_LOCK_ROW_CONFLICT - if lock conflict, need rollback transaction outside + */ + int lock_row(const uint64_t table_id, const common::ObString& key, ObHTableLockMode mode, ObHTableLockHandle &handle); + + /** + * release_handle - release the htable lock handle + * + * this is the end of lifecycle of htable lock inside a transaction + * the HTableLockHandle object should not been access anymore after release. + * + * @handle: the target htable lock handle + * + * Returns: + * OB_SUCCESS - OK + */ + int release_handle(ObHTableLockHandle &handle); + static int mtl_init(ObHTableLockMgr *&htable_lock_mgr); + static void mtl_destroy(ObHTableLockMgr *&htable_lock_mgr); +private: + int init(); + int alloc_lock_node(ObHTableLockMode lock_mode, ObHTableLockNode *&lock_node); + int alloc_lock_key(const uint64_t table_id, const common::ObString &key, ObHTableLockKey *&lock_key); + int alloc_lock(ObHTableLockMode lock_mode, ObHTableLock *&lock); + int internal_lock_row(const uint64_t table_id, const common::ObString& key, ObHTableLockMode lock_mode, ObHTableLockHandle &handle); + // escalate read lock to write lock + int rd2wrlock(ObHTableLockNode &lock_node); + int release_node(ObHTableLockNode &lock_node); +private: + static const uint64_t DEFAULT_BUCKET_NUM = 20480; + typedef common::hash::ObHashMap ObHTableLockMap; + + bool is_inited_; + common::SpinRWLock spin_lock_; + ObHTableLockMap lock_map_; + // alloc memory for ObHTableLockKey, ObHTableLockHandle, ObHTableLock, need free + common::ObFIFOAllocator allocator_; + DISALLOW_COPY_AND_ASSIGN(ObHTableLockMgr); +}; + +// execute lock operation +class ObHTableLockOp +{ +public: + explicit ObHTableLockOp(ObHTableLockMode lock_mode) + : lock_mode_(lock_mode), + old_lock_key_(nullptr), + is_called_(false), + ret_code_(common::OB_SUCCESS) + {} + virtual ~ObHTableLockOp() {} + void operator() (common::hash::HashMapPair &entry); + ObHTableLockKey *get_lock_key() { return old_lock_key_; } + bool is_called() { return is_called_; } + int get_ret() { return ret_code_; } + +private: + ObHTableLockMode lock_mode_; + ObHTableLockKey *old_lock_key_; + bool is_called_; // indicate whether this op was called or not + bool lock_success_; // indicate whether lock op is success or not + int ret_code_; + DISALLOW_COPY_AND_ASSIGN(ObHTableLockOp); +}; + +// execute unlock operation firstly, and erase the lock pair from the map if no one hold it anymore +class ObHTableUnLockOpPred +{ +public: + explicit ObHTableUnLockOpPred() : ret_code_(common::OB_SUCCESS) {} + virtual ~ObHTableUnLockOpPred() {} + bool operator() (common::hash::HashMapPair &entry); + int get_ret() { return ret_code_; } + +private: + int ret_code_; + DISALLOW_COPY_AND_ASSIGN(ObHTableUnLockOpPred); +}; + +// try to execute lock escalation operation +class ObHTableRd2WrLockOp +{ +public: + ObHTableRd2WrLockOp() : ret_code_(common::OB_SUCCESS) {} + virtual ~ObHTableRd2WrLockOp() {} + void operator() (common::hash::HashMapPair &entry); + int get_ret() { return ret_code_; } + +private: + int ret_code_; + DISALLOW_COPY_AND_ASSIGN(ObHTableRd2WrLockOp); +}; + +#define HTABLE_LOCK_MGR (MTL(ObHTableLockMgr*)) + +} // end namespace table +} // end namespace oceanbase + +#endif diff --git a/src/observer/table/ob_htable_utils.cpp b/src/observer/table/ob_htable_utils.cpp index 7c8db191b7..eea0087353 100644 --- a/src/observer/table/ob_htable_utils.cpp +++ b/src/observer/table/ob_htable_utils.cpp @@ -13,8 +13,12 @@ #define USING_LOG_PREFIX SERVER #include "ob_htable_utils.h" #include // be64toh +#include "ob_htable_filters.h" +#include "ob_htable_filter_operator.h" +#include "share/table/ob_table.h" using namespace oceanbase::common; using namespace oceanbase::table; +using namespace oceanbase::share::schema; ObHTableCellEntity::ObHTableCellEntity(common::ObNewRow *ob_row) :ob_row_(ob_row) @@ -98,6 +102,18 @@ ObString ObHTableCellEntity2::get_value() const } return rowkey_str; } + +int ObHTableCellEntity2::get_value(ObString &str) const +{ + int ret = OB_SUCCESS; + ObObj val; + if (OB_FAIL(entity_->get_property(ObHTableConstants::VALUE_CNAME_STR, val))) { + LOG_WARN("failed to get property K", K(ret)); + } else { + str = val.get_varchar(); + } + return ret; +} //////////////////////////////////////////////////////////////// ObString ObHTableCellEntity3::get_rowkey() const { @@ -310,3 +326,91 @@ int ObHTableUtils::int64_to_java_bytes(int64_t val, char bytes[8]) memcpy(bytes, &big_endian_64bits, sizeof(int64_t)); return OB_SUCCESS; } + +int ObHTableUtils::lock_htable_rows(uint64_t table_id, const ObTableBatchOperation &mutations, ObHTableLockHandle &handle, ObHTableLockMode lock_mode) +{ + int ret = OB_SUCCESS; + const int64_t N = mutations.count(); + if (table_id == OB_INVALID_ID) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table id", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + const ObITableEntity &entity = mutations.at(i).entity(); + ObHTableCellEntity3 htable_cell(&entity); + ObString row = htable_cell.get_rowkey(); + if (row.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null htable rowkey", K(ret)); + } else if (OB_FAIL(HTABLE_LOCK_MGR->lock_row(table_id, row, lock_mode, handle))) { + LOG_WARN("fail to lock htable row", K(ret), K(table_id), K(row), K(lock_mode)); + } + } + return ret; +} + +int ObHTableUtils::lock_htable_row(uint64_t table_id, const ObTableQuery &htable_query, ObHTableLockHandle &handle, ObHTableLockMode lock_mode) +{ + int ret = OB_SUCCESS; + if (table_id == OB_INVALID_ID) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table id", K(ret)); + } else if (!htable_query.get_htable_filter().is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid query", K(ret)); + } else { + const ObIArray &key_ranges = htable_query.get_scan_ranges(); + const ObObj *start_key_ptr = key_ranges.at(0).start_key_.get_obj_ptr(); + const ObObj *end_key_ptr = key_ranges.at(0).end_key_.get_obj_ptr(); + if (OB_ISNULL(start_key_ptr) || OB_ISNULL(end_key_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(ret), KP(start_key_ptr), KP(end_key_ptr), K(key_ranges.at(0))); + } else { + ObString start_key = start_key_ptr->get_string(); + ObString end_key = end_key_ptr->get_string(); + if (start_key.empty() || end_key.empty() || start_key != end_key) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument, check operation must only has one row", K(ret), K(start_key), K(end_key)); + } else if (OB_FAIL(HTABLE_LOCK_MGR->lock_row(table_id, start_key, lock_mode, handle))) { + LOG_WARN("fail to lock htable row", K(ret), K(table_id), K(start_key), K(lock_mode)); + } + } + } + return ret; +} +int ObHTableUtils::check_htable_schema(const ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + const ObColumnSchemaV2 *rowkey_schema = NULL; + const ObColumnSchemaV2 *qualifier_schema = NULL; + const ObColumnSchemaV2 *version_schema = NULL; + const ObColumnSchemaV2 *value_schema = NULL; + + if (OB_ISNULL(rowkey_schema = table_schema.get_column_schema_by_idx(ObHTableConstants::COL_IDX_K))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("can't get rowkey column schema"); + } else if (ObHTableConstants::ROWKEY_CNAME_STR.case_compare(rowkey_schema->get_column_name()) != 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the first column should be K", K(ret), K(rowkey_schema->get_column_name())); + } else if (OB_ISNULL(qualifier_schema = table_schema.get_column_schema_by_idx(ObHTableConstants::COL_IDX_Q))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("can't get qualifier column schema"); + } else if (ObHTableConstants::CQ_CNAME_STR.case_compare(qualifier_schema->get_column_name()) != 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the second column should be Q", K(ret), K(qualifier_schema->get_column_name())); + } else if (OB_ISNULL(version_schema = table_schema.get_column_schema_by_idx(ObHTableConstants::COL_IDX_T))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("can't get version column schema"); + } else if (ObHTableConstants::VERSION_CNAME_STR.case_compare(version_schema->get_column_name()) != 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the third column should be T", K(ret), K(version_schema->get_column_name())); + } else if (OB_ISNULL(value_schema = table_schema.get_column_schema_by_idx(ObHTableConstants::COL_IDX_V))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("can't get version column schema"); + } else if (ObHTableConstants::VALUE_CNAME_STR.case_compare(value_schema->get_column_name()) != 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the fourth column should be V", K(ret), K(value_schema->get_column_name())); + } + + return ret; +} diff --git a/src/observer/table/ob_htable_utils.h b/src/observer/table/ob_htable_utils.h index bd9b6ea534..5547b9e5bb 100644 --- a/src/observer/table/ob_htable_utils.h +++ b/src/observer/table/ob_htable_utils.h @@ -15,7 +15,10 @@ #include "common/row/ob_row.h" #include "lib/string/ob_string.h" #include "share/table/ob_table.h" +#include "ob_htable_lock_mgr.h" +#include "share/table/ob_table_rpc_struct.h" #include +#include "share/schema/ob_table_schema.h" namespace oceanbase { @@ -160,6 +163,7 @@ public: virtual int64_t get_timestamp() const override; virtual common::ObString get_value() const override; virtual Type get_type() const { return Type::NORMAL; } + int get_value(ObString &str) const; private: const ObITableEntity *entity_; DISALLOW_COPY_AND_ASSIGN(ObHTableCellEntity2); @@ -302,6 +306,13 @@ public: static int64_t current_time_millis() { return common::ObTimeUtility::current_time() / 1000; } static int java_bytes_to_int64(const ObString &bytes, int64_t &val); static int int64_to_java_bytes(int64_t val, char bytes[8]); + // lock all rows of mutations in the given lock mode with the given lock handle, + // for put, delete, mutations in check_and_xxx + static int lock_htable_rows(uint64_t table_id, const ObTableBatchOperation &mutations, ObHTableLockHandle &handle, ObHTableLockMode lock_mode); + // lock the check row in the given lock mode with the given lock hanle, + // for increment, append, and check operation in check_and_xxx + static int lock_htable_row(uint64_t table_id, const ObTableQuery &htable_query, ObHTableLockHandle &handle, ObHTableLockMode lock_mode); + static int check_htable_schema(const share::schema::ObTableSchema &table_schema); private: ObHTableUtils() = delete; ~ObHTableUtils() = delete; diff --git a/src/observer/table/ob_table_batch_execute_processor.cpp b/src/observer/table/ob_table_batch_execute_processor.cpp index 18faa4d4c8..770d1a0f95 100644 --- a/src/observer/table/ob_table_batch_execute_processor.cpp +++ b/src/observer/table/ob_table_batch_execute_processor.cpp @@ -306,20 +306,27 @@ int ObTableBatchExecuteP::htable_put() const ObTableBatchOperation &batch_operation = arg_.batch_operation_; observer::ObReqTimeGuard req_timeinfo_guard; // 引用cache资源必须加ObReqTimeGuard ObTableApiCacheGuard cache_guard; + ObHTableLockHandle *lock_handle = nullptr; + uint64_t table_id = OB_INVALID_ID; if (OB_FAIL(check_arg2())) { LOG_WARN("fail to check arg", K(ret)); } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { LOG_WARN("fail to init table ctx", K(ret)); + } else if (FALSE_IT(table_id = tb_ctx_.get_table_id())) { } else if (OB_FAIL(start_trans(false, /* is_readonly */ sql::stmt::T_INSERT, arg_.consistency_level_, - tb_ctx_.get_table_id(), + table_id, tb_ctx_.get_ls_id(), get_timeout_ts()))) { LOG_WARN("failed to start readonly transaction", K(ret)); } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(HTABLE_LOCK_MGR->acquire_handle(get_trans_desc()->tid(), lock_handle))) { + LOG_WARN("fail to get htable lock handle", K(ret)); + } else if (OB_FAIL(ObHTableUtils::lock_htable_rows(table_id, batch_operation, *lock_handle, ObHTableLockMode::SHARED))) { + LOG_WARN("fail to lock htable rows", K(ret), K(table_id), K(batch_operation)); } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx_, cache_guard, spec))) { @@ -350,8 +357,9 @@ int ObTableBatchExecuteP::htable_put() } int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("fail to end trans"); + const bool use_sync = false; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts(), use_sync, lock_handle))) { + LOG_WARN("failed to end trans"); } ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; @@ -489,20 +497,27 @@ int ObTableBatchExecuteP::htable_delete() tb_ctx_.set_batch_operation(&batch_operation); observer::ObReqTimeGuard req_timeinfo_guard; // 引用cache资源必须加ObReqTimeGuard ObTableApiCacheGuard cache_guard; + ObHTableLockHandle *lock_handle = nullptr; + uint64_t table_id = OB_INVALID_ID; if (OB_FAIL(check_arg2())) { LOG_WARN("fail to check arg", K(ret)); } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { LOG_WARN("fail to init table ctx", K(ret)); + } else if (FALSE_IT(table_id = tb_ctx_.get_table_id())) { } else if (OB_FAIL(start_trans(false, /* is_readonly */ sql::stmt::T_INSERT, arg_.consistency_level_, - tb_ctx_.get_table_id(), + table_id, tb_ctx_.get_ls_id(), get_timeout_ts()))) { LOG_WARN("failed to start readonly transaction", K(ret)); } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(HTABLE_LOCK_MGR->acquire_handle(get_trans_desc()->tid(), lock_handle))) { + LOG_WARN("fail to get htable lock handle", K(ret)); + } else if (OB_FAIL(ObHTableUtils::lock_htable_rows(table_id, batch_operation, *lock_handle, ObHTableLockMode::SHARED))) { + LOG_WARN("fail to lock htable rows", K(ret), K(table_id), K(batch_operation)); } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx_, cache_guard, spec))) { @@ -541,10 +556,10 @@ int ObTableBatchExecuteP::htable_delete() LOG_WARN("failed to add result", K(ret)); } } - + const bool use_sync = false; int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("fail to end trans"); + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts(), use_sync, lock_handle))) { + LOG_WARN("failed to end trans"); } ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; return ret; @@ -880,9 +895,11 @@ int ObTableBatchExecuteP::htable_mutate_row() { int ret = OB_SUCCESS; const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; + const ObTableBatchOperation &batch_operation = arg_.batch_operation_; uint64_t table_id = OB_INVALID_ID; ObSEArray tablet_ids; ObLSID ls_id; + ObHTableLockHandle *lock_handle = nullptr; if (OB_FAIL(check_arg2())) { LOG_WARN("fail to check arg", K(ret)); @@ -902,8 +919,11 @@ int ObTableBatchExecuteP::htable_mutate_row() ls_id, get_timeout_ts()))) { LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(HTABLE_LOCK_MGR->acquire_handle(get_trans_desc()->tid(), lock_handle))) { + LOG_WARN("fail to get htable lock handle", K(ret)); + } else if (OB_FAIL(ObHTableUtils::lock_htable_rows(table_id, batch_operation, *lock_handle, ObHTableLockMode::SHARED))) { + LOG_WARN("fail to lock htable rows", K(ret), K(table_id), K(batch_operation)); } else { - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; int64_t N = batch_operation.count(); for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { @@ -939,7 +959,8 @@ int ObTableBatchExecuteP::htable_mutate_row() } int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + const bool use_sync = false; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts(), use_sync, lock_handle))) { LOG_WARN("failed to end trans"); } ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; diff --git a/src/observer/table/ob_table_cache.cpp b/src/observer/table/ob_table_cache.cpp index 78953d332f..32fcf0b817 100644 --- a/src/observer/table/ob_table_cache.cpp +++ b/src/observer/table/ob_table_cache.cpp @@ -25,6 +25,7 @@ int ObTableApiCacheKey::deep_copy(common::ObIAllocator &allocator, const ObILibC table_id_ = table_key.table_id_; index_table_id_ = table_key.index_table_id_; schema_version_ = table_key.schema_version_; + is_ttl_table_ = table_key.is_ttl_table_; operation_type_ = table_key.operation_type_; namespace_ = table_key.namespace_; for (int64_t i = 0; OB_SUCC(ret) && i < table_key.op_column_ids_.count(); i++) { @@ -41,6 +42,7 @@ uint64_t ObTableApiCacheKey::hash() const hash_val = murmurhash(&table_id_, sizeof(table_id_), hash_val); hash_val = murmurhash(&index_table_id_, sizeof(index_table_id_), hash_val); hash_val = murmurhash(&schema_version_, sizeof(schema_version_), hash_val); + hash_val = murmurhash(&is_ttl_table_, sizeof(is_ttl_table_), hash_val); hash_val = murmurhash(&operation_type_, sizeof(operation_type_), hash_val); for (int64_t i = 0; i < op_column_ids_.count(); i++) { hash_val = murmurhash(&(op_column_ids_.at(i)), sizeof(uint64_t), hash_val); @@ -54,6 +56,7 @@ bool ObTableApiCacheKey::is_equal(const ObILibCacheKey &other) const bool cmp_ret = table_id_ == table_key.table_id_ && index_table_id_ == table_key.index_table_id_ && schema_version_ == table_key.schema_version_ && + is_ttl_table_ == table_key.is_ttl_table_ && operation_type_ == table_key.operation_type_ && namespace_ == table_key.namespace_ && op_column_ids_.count() == table_key.op_column_ids_.count(); @@ -70,6 +73,7 @@ void ObTableApiCacheKey::reset() table_id_ = common::OB_INVALID_ID; index_table_id_ = common::OB_INVALID_ID; schema_version_ = -1; + is_ttl_table_ = false; operation_type_ = ObTableOperationType::Type::INVALID; namespace_ = ObLibCacheNameSpace::NS_TABLEAPI; op_column_ids_.reset(); @@ -131,6 +135,7 @@ int ObTableApiCacheGuard::create_cache_key(ObTableCtx *tb_ctx) cache_key_.table_id_ = tb_ctx->get_table_id(); cache_key_.index_table_id_ = tb_ctx->get_index_table_id(); cache_key_.schema_version_ = tb_ctx->get_table_schema()->get_schema_version(); + cache_key_.is_ttl_table_ = tb_ctx->is_ttl_table(); ObTableOperationType::Type operation_type = tb_ctx->get_opertion_type(); cache_key_.operation_type_ = operation_type; if (operation_type == ObTableOperationType::Type::UPDATE diff --git a/src/observer/table/ob_table_cache.h b/src/observer/table/ob_table_cache.h index df78c53fd6..9ad0e9f596 100644 --- a/src/observer/table/ob_table_cache.h +++ b/src/observer/table/ob_table_cache.h @@ -34,18 +34,8 @@ struct ObTableApiCacheKey: public ObILibCacheKey table_id_(common::OB_INVALID_ID), index_table_id_(common::OB_INVALID_ID), schema_version_(-1), - operation_type_(ObTableOperationType::Type::INVALID) - {} - ObTableApiCacheKey(common::ObTableID table_id, - common::ObTableID index_table_id, - int64_t schema_version, - ObTableOperationType::Type operation_type, - uint8_t extend_type = 0) - : ObILibCacheKey(ObLibCacheNameSpace::NS_TABLEAPI), - table_id_(table_id), - index_table_id_(index_table_id), - schema_version_(schema_version), - operation_type_(operation_type) + operation_type_(ObTableOperationType::Type::INVALID), + is_ttl_table_(false) {} void reset(); virtual int deep_copy(common::ObIAllocator &allocator, const ObILibCacheKey &other); @@ -54,6 +44,7 @@ struct ObTableApiCacheKey: public ObILibCacheKey TO_STRING_KV(K_(table_id), K_(schema_version), + K_(is_ttl_table), K_(operation_type), K_(index_table_id), K_(op_column_ids), @@ -63,6 +54,7 @@ struct ObTableApiCacheKey: public ObILibCacheKey common::ObTableID index_table_id_; int64_t schema_version_; ObTableOperationType::Type operation_type_; + bool is_ttl_table_; common::ObArray op_column_ids_; }; diff --git a/src/observer/table/ob_table_cg_service.cpp b/src/observer/table/ob_table_cg_service.cpp index bf51d94345..87814cc49a 100644 --- a/src/observer/table/ob_table_cg_service.cpp +++ b/src/observer/table/ob_table_cg_service.cpp @@ -21,6 +21,7 @@ namespace oceanbase { namespace table { + ObRawExpr* ObTableExprCgService::get_ref_raw_expr(const ObIArray &all_exprs, const ObString &col_name) { @@ -202,6 +203,24 @@ int ObTableExprCgService::generate_column_ref_raw_expr(ObTableCtx &ctx, return ret; } +int ObTableExprCgService::genreate_filter_exprs(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + + // won't put filter exprs into all exprs here + if (ctx.is_ttl_table()) { + ObIArray &filter_exprs = ctx.get_filter_exprs(); + ObRawExpr *expire_expr = nullptr; + if (OB_FAIL(build_expire_expr(ctx, expire_expr))) { + LOG_WARN("fail to build expire expr", K(ret), K(ctx)); + } else if(OB_FAIL(filter_exprs.push_back(expire_expr))) { + LOG_WARN("fail to push back expire expr", K(ret), K(filter_exprs)); + } + } + + return ret; +} + int ObTableExprCgService::generate_exprs(ObTableCtx &ctx, oceanbase::common::ObIAllocator &allocator, oceanbase::sql::ObExprFrameInfo &expr_frame_info) @@ -216,10 +235,13 @@ int ObTableExprCgService::generate_exprs(ObTableCtx &ctx, } if (OB_SUCC(ret)) { - if (OB_FAIL(generate_expr_frame_info(ctx, allocator, expr_frame_info))) { + if (OB_FAIL(genreate_filter_exprs(ctx))) { + LOG_WARN("fail to generate filter exprs", K(ret), K(ctx)); + } else if (OB_FAIL(generate_expr_frame_info(ctx, allocator, expr_frame_info))) { LOG_WARN("fail to generate expr frame info", K(ret), K(ctx)); } } + return ret; } @@ -227,7 +249,7 @@ int ObTableExprCgService::generate_column_raw_exprs(ObTableCtx &ctx) { int ret = OB_SUCCESS; const ObTableSchema *table_schema = ctx.get_table_schema(); - ObIArray *exprs = (ctx.is_for_update() || ctx.is_for_insertup())? + ObIArray *exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? &ctx.get_old_row_exprs() : const_cast *>(&ctx.get_all_exprs().get_expr_array()); if (OB_ISNULL(table_schema)) { @@ -258,6 +280,73 @@ int ObTableExprCgService::generate_column_raw_exprs(ObTableCtx &ctx) return ret; } +int ObTableExprCgService::build_expire_expr(ObTableCtx &ctx, ObRawExpr *&expire_expr) +{ + int ret = OB_SUCCESS; + const ObIArray &exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObRawExprFactory &expr_factory = ctx.get_expr_factory(); + ObSysFunRawExpr *now_func_expr = nullptr; + ObOpRawExpr *expire_expr_tmp = nullptr; + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret), K(ctx)); + } else if (OB_FAIL(expr_factory.create_raw_expr(T_FUN_SYS_CUR_TIMESTAMP, now_func_expr))) { + LOG_WARN("fail to create current timestamp expr", K(ret), K(expr_factory)); + } else if (OB_FAIL(expr_factory.create_raw_expr(T_OP_LE, expire_expr_tmp))) { + LOG_WARN("fail to create T_OP_LE expr", K(ret), K(expr_factory)); + } else { + const ObString &ttl_definition = table_schema->get_ttl_definition(); + ObArray columns; + ObSchemaChecker schema_checker; + ObSchemaGetterGuard &schema_guard = ctx.get_schema_guard(); + ObSQLSessionInfo &sess_info = ctx.get_session_info(); + ObRawExpr *ttl_gen_expr = nullptr; + if (OB_FAIL(schema_checker.init(schema_guard))) { + LOG_WARN("fail to init schema checker", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_generated_column_expr(ttl_definition, + expr_factory, + sess_info, + ttl_gen_expr, + columns, + table_schema, + false, /* allow_sequence */ + nullptr, + &schema_checker))) { + LOG_WARN("fail to build expire expr", K(ret), K(ttl_definition)); + } else { + // 找到生成列引用的列并替换为真正的列 + ObRawExpr *real_ref_expr = nullptr; + const ObIArray *src_exprs = &exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { + if (OB_ISNULL(real_ref_expr = get_ref_raw_expr(*src_exprs, columns.at(i).col_name_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get ref expr", K(ret), K(columns.at(i).ref_expr_->get_column_id())); + } else if (OB_FAIL(ObRawExprUtils::replace_ref_column(ttl_gen_expr, + columns.at(i).ref_expr_, + real_ref_expr))) { + LOG_WARN("fail to replace column reference expr", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(expire_expr_tmp->set_param_exprs(ttl_gen_expr, now_func_expr))) { + LOG_WARN("fail to set expire expr param exprs", K(ret), K(*ttl_gen_expr), K(*now_func_expr)); + } else if (OB_FAIL(expire_expr_tmp->formalize(&sess_info))) { + LOG_WARN("fail to formailize expire expr", K(ret)); + } else { + expire_expr = expire_expr_tmp; + } + } + } + } + + return ret; +} + + int ObTableExprCgService::generate_full_assign_raw_exprs(ObTableCtx &ctx) { int ret = OB_SUCCESS; @@ -412,8 +501,8 @@ int ObTableExprCgService::generate_update_raw_exprs(ObTableCtx &ctx) } int ObTableExprCgService::generate_expr_frame_info(ObTableCtx &ctx, - common::ObIAllocator &allocator, - ObExprFrameInfo &expr_frame_info) + common::ObIAllocator &allocator, + ObExprFrameInfo &expr_frame_info) { int ret = OB_SUCCESS; ObStaticEngineExprCG expr_cg(allocator, @@ -422,9 +511,22 @@ int ObTableExprCgService::generate_expr_frame_info(ObTableCtx &ctx, 0, 0, ctx.get_cur_cluster_version()); - if (OB_FAIL(expr_cg.generate(ctx.get_all_exprs(), expr_frame_info))) { - LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); + + if (ctx.get_filter_exprs().empty()) { + if (OB_FAIL(expr_cg.generate(ctx.get_all_exprs(), expr_frame_info))) { + LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); + } + } else { + ObRawExprUniqueSet exprs(false/* need_unique */); + if (OB_FAIL(exprs.append(ctx.get_all_exprs().get_expr_array()))) { + LOG_WARN("fail to append all exprs", K(ret), K(ctx.get_all_exprs().get_expr_array())); + } else if (OB_FAIL(exprs.append(ctx.get_filter_exprs()))) { + LOG_WARN("fail to append filter exprs", K(ret), K(ctx.get_filter_exprs())); + } else if (OB_FAIL(expr_cg.generate(exprs, expr_frame_info))) { + LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); + } } + return ret; } @@ -540,6 +642,14 @@ int ObTableExprCgService::refresh_update_exprs_frame(ObTableCtx &ctx, return ret; } +int ObTableExprCgService::refresh_ttl_exprs_frame(ObTableCtx &ctx, + const ObIArray &ins_new_row, + const ObIArray &delta_exprs, + const ObTableEntity &entity) +{ + return refresh_insert_up_exprs_frame(ctx, ins_new_row, delta_exprs, entity); +} + int ObTableExprCgService::refresh_insert_up_exprs_frame(ObTableCtx &ctx, const ObIArray &ins_new_row, const ObIArray &delta_exprs, @@ -1290,9 +1400,9 @@ int ObTableDmlCgService::generate_delete_ctdef(ObTableCtx &ctx, int ret = OB_SUCCESS; ObSEArray old_row; ObSEArray new_row; - - const ObIArray &exprs = ctx.get_all_exprs().get_expr_array(); ObSEArray tmp_exprs; + const ObIArray &exprs = ctx.get_all_exprs().get_expr_array(); + if (OB_FAIL(replace_exprs_with_dependant(exprs, tmp_exprs))) { LOG_WARN("fail to replace exprs with dependant", K(ret)); } else if (OB_FAIL(old_row.assign(tmp_exprs))) { @@ -1469,6 +1579,7 @@ int ObTableDmlCgService::generate_tsc_ctdef(ObTableCtx &ctx, const ObIArray &all_exprs = ctx.is_for_insertup() ? ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); ObSEArray access_exprs; + ObIArray &filter_exprs = ctx.get_filter_exprs(); const ObTableSchema *table_schema = ctx.get_table_schema(); tsc_ctdef.ref_table_id_ = ctx.get_index_table_id(); const uint64_t tenant_id = MTL_ID(); @@ -1703,7 +1814,33 @@ int ObTableDmlCgService::generate_insert_up_ctdef(ObTableCtx &ctx, allocator, assign_ids, ins_up_ctdef.upd_ctdef_))) { + LOG_WARN("fail to generate update ctdef", K(ret), K(ctx)); + } + + return ret; +} + +int ObTableDmlCgService::generate_ttl_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableTTLCtDef &ttl_ctdef) +{ + int ret = OB_SUCCESS; + ObIArray &filter_exprs = ctx.get_filter_exprs(); + ObStaticEngineCG cg(ctx.get_cur_cluster_version()); + + if (OB_FAIL(generate_insert_ctdef(ctx, allocator, ttl_ctdef.ins_ctdef_))) { + LOG_WARN("fail to generate insert ctdef", K(ret), K(ctx)); + } else if (OB_FAIL(generate_table_rowkey_info(ctx, ttl_ctdef.ins_ctdef_))) { + LOG_WARN("fail to generate table rowkey info", K(ret), K(ctx)); + } else if (OB_FAIL(generate_delete_ctdef(ctx, allocator, ttl_ctdef.del_ctdef_))) { LOG_WARN("fail to generate delete ctdef", K(ret), K(ctx)); + } else if (OB_FAIL(generate_update_ctdef(ctx, allocator, ctx.get_assign_ids(), ttl_ctdef.upd_ctdef_))) { + LOG_WARN("fail to generate update ctdef", K(ret), K(ctx)); + } else if (filter_exprs.count() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid filter exprs count", K(ret), K(filter_exprs)); + } else if (OB_FAIL(cg.generate_rt_expr(*filter_exprs.at(0), ttl_ctdef.expire_expr_))) { + LOG_WARN("fail to generate expire rt expr", K(ret)); } return ret; @@ -2404,12 +2541,19 @@ int ObTableTscCgService::generate_tsc_ctdef(const ObTableCtx &ctx, ObTableApiScanCtDef &tsc_ctdef) { int ret = OB_SUCCESS; + ObStaticEngineCG cg(ctx.get_cur_cluster_version()); + const int64_t filter_exprs_cnt = ctx.get_filter_exprs().count(); + // init scan_ctdef_.ref_table_id_ tsc_ctdef.scan_ctdef_.ref_table_id_ = ctx.get_index_table_id(); if (OB_FAIL(tsc_ctdef.output_exprs_.init(ctx.get_select_exprs().count()))) { - LOG_WARN("fail to init output expr", K(ret)); + LOG_WARN("fail to init output exprs", K(ret)); + } else if (filter_exprs_cnt != 0 && OB_FAIL(tsc_ctdef.filter_exprs_.init(ctx.get_filter_exprs().count()))) { + LOG_WARN("fail to init filter exprs", K(ret)); } else if (OB_FAIL(generate_output_exprs(ctx, tsc_ctdef.output_exprs_))) { LOG_WARN("fail to generate output exprs", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(ctx.get_filter_exprs(), tsc_ctdef.filter_exprs_))) { + LOG_WARN("fail to generate filter rt exprs ", K(ret)); } else if (OB_FAIL(generate_das_tsc_ctdef(ctx, allocator, tsc_ctdef.scan_ctdef_))) { // init scan_ctdef_ LOG_WARN("fail to generate das scan ctdef", K(ret)); } else if (ctx.is_index_back()) { @@ -2440,7 +2584,11 @@ int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, ObTableCtx &ctx, ObTableApiInsertSpec &spec) { - return ObTableDmlCgService::generate_insert_ctdef(ctx, alloc, spec.get_ctdef()); + int ret = OB_SUCCESS; + if (OB_FAIL(ObTableDmlCgService::generate_insert_ctdef(ctx, alloc, spec.get_ctdef()))) { + LOG_WARN("fail to generate ctdef", KR(ret)); + } + return ret; } int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, @@ -2509,8 +2657,7 @@ int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, alloc, spec.get_conflict_checker_ctdef()))) { LOG_WARN("fail to generate conflict checker ctdef", K(ret)); - } else if (OB_FAIL(cg.generate_rt_exprs(exprs, - spec.get_all_saved_exprs()))) { + } else if (OB_FAIL(cg.generate_rt_exprs(exprs, spec.get_all_saved_exprs()))) { LOG_WARN("fail to generate rt exprs ", K(ret)); } } @@ -2560,5 +2707,20 @@ int ObTableExprCgService::generate_autoinc_nextval_expr(ObTableCtx &ctx, return ret; } +int ObTableSpecCgService::generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiTTLSpec &spec) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableDmlCgService::generate_ttl_ctdef(ctx, alloc, spec.get_ctdef()))) { + LOG_WARN("fail to generate ttl ctdef", K(ret)); + } else if (OB_FAIL(ObTableDmlCgService::generate_conflict_checker_ctdef(ctx, alloc, spec.get_conflict_checker_ctdef()))) { + LOG_WARN("fail to generate conflict checker ctdef", K(ret)); + } + + return ret; +} + } // namespace table } // namespace oceanbase diff --git a/src/observer/table/ob_table_cg_service.h b/src/observer/table/ob_table_cg_service.h index bc22e12983..67582ef09c 100644 --- a/src/observer/table/ob_table_cg_service.h +++ b/src/observer/table/ob_table_cg_service.h @@ -55,6 +55,10 @@ public: static int refresh_replace_exprs_frame(ObTableCtx &ctx, const common::ObIArray &exprs, const ObTableEntity &entity); + static int refresh_ttl_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &ins_new_row, + const common::ObIArray &delta_exprs, + const ObTableEntity &entity); static int refresh_update_exprs_frame(ObTableCtx &ctx, const common::ObIArray &old_row, const common::ObIArray &new_row, @@ -95,6 +99,7 @@ private: const common::ObIArray &delta_exprs, const ObTableEntity &entity); static int generate_full_assign_raw_exprs(ObTableCtx &ctx); + static int genreate_filter_exprs(ObTableCtx &ctx); private: // 通过column_name在表达式数组获取列引用表达式 static ObRawExpr* get_ref_raw_expr(const common::ObIArray &all_exprs, @@ -115,6 +120,7 @@ private: ObRawExpr *&expr, const ObColumnSchemaV2 &col_schema); + static int build_expire_expr(ObTableCtx &ctx, sql::ObRawExpr *&expire_expr); static int write_datum(ObTableCtx &ctx, common::ObIAllocator &allocator, const sql::ObExpr &expr, @@ -156,6 +162,9 @@ public: ObTableInsUpdCtDef &ins_up_ctdef); static int generate_lock_ctdef(ObTableCtx &ctx, ObTableLockCtDef &lock_ctdef); + static int generate_ttl_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableTTLCtDef &ttl_ctdef); static int generate_conflict_checker_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, sql::ObConflictCheckerCtdef &conflict_checker_ctdef); @@ -268,9 +277,9 @@ public: ret = ObTableSpecCgService::generate_with_child (alloc, ctx, root_spec); } else if (OB_FAIL(ObTableExecutorFactory::generate_spec(alloc, - static_cast(TYPE), - ctx, - spec))) { + static_cast(TYPE), + ctx, + spec))) { SERVER_LOG(WARN, "fail to generate spec", K(ret)); } else { root_spec = spec; @@ -307,6 +316,10 @@ public: ObTableCtx &ctx, ObTableApiLockSpec &spec); + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiTTLSpec &spec); + private: template static int generate_with_child(common::ObIAllocator &alloc, diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index 90d865a5de..6fa7948aa5 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -66,6 +66,30 @@ int ObTableCtx::init_sess_info(ObTableApiCredential &credential) ret = OB_ERR_UNEXPECTED; LOG_WARN("session info is null", K(ret), K(credential)); } + return ret; +} + +int ObTableCtx::init_common(ObTableApiCredential &credential, + const common::ObTabletID &arg_tablet_id, + const uint64_t table_id, + const int64_t &timeout_ts) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = credential.tenant_id_; + const uint64_t database_id = credential.database_id_; + + if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard_))) { + LOG_WARN("fail to get schema guard", K(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard_.get_table_schema(tenant_id, + table_id, + table_schema_))) { + LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(table_id)); + } else if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNKNOWN_TABLE; + LOG_WARN("fail get table schema by table id", K(ret), K(tenant_id), K(database_id), K(table_id)); + } else if (OB_FAIL(inner_init_common(credential, arg_tablet_id, table_schema_->get_table_name(), timeout_ts))) { + LOG_WARN("fail to inner init common", KR(ret), K(credential), K(arg_tablet_id), K(timeout_ts)); + } return ret; } @@ -76,11 +100,8 @@ int ObTableCtx::init_common(ObTableApiCredential &credential, const int64_t &timeout_ts) { int ret = OB_SUCCESS; - bool is_cache_hit = false; - const ObTenantSchema *tenant_schema = nullptr; const uint64_t tenant_id = credential.tenant_id_; const uint64_t database_id = credential.database_id_; - ObTabletID tablet_id = arg_tablet_id; if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard_))) { LOG_WARN("fail to get schema guard", K(ret), K(tenant_id), K(arg_table_name)); @@ -89,10 +110,29 @@ int ObTableCtx::init_common(ObTableApiCredential &credential, arg_table_name, false, /* is_index */ table_schema_))) { - LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(database_id), K(arg_table_name)); - } else if (OB_ISNULL(table_schema_)) { + LOG_WARN("fail to get table schema", K(ret), K(tenant_id), "database_id", credential.database_id_, K(arg_table_name)); + } else if (OB_FAIL(inner_init_common(credential, arg_tablet_id, arg_table_name, timeout_ts))) { + LOG_WARN("fail to inner init common", KR(ret), K(credential), K(arg_tablet_id), + K(arg_table_name), K(timeout_ts)); + } + + return ret; +} + +int ObTableCtx::inner_init_common(ObTableApiCredential &credential, + const common::ObTabletID &arg_tablet_id, + const common::ObString &table_name, + const int64_t &timeout_ts) +{ + int ret = OB_SUCCESS; + bool is_cache_hit = false; + const ObTenantSchema *tenant_schema = nullptr; + const uint64_t tenant_id = credential.tenant_id_; + const uint64_t database_id = credential.database_id_; + ObTabletID tablet_id = arg_tablet_id; + if (OB_ISNULL(table_schema_)) { ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("fail get table schema by table name", K(ret), K(tenant_id), K(database_id), K(arg_table_name)); + LOG_WARN("table schema is null", K(ret), K(tenant_id), K(table_name)); } else if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id, tenant_schema))) { LOG_WARN("fail to get tenant schema", K(ret), K(tenant_id)); } else if (OB_ISNULL(tenant_schema)) { @@ -104,7 +144,7 @@ int ObTableCtx::init_common(ObTableApiCredential &credential, if (is_scan_) { // 扫描场景使用table_schema上的tablet id,客户端已经做了路由分发 if (table_schema_->is_partitioned_table()) { // 不支持分区表 ret = OB_NOT_SUPPORTED; - LOG_WARN("partitioned table not supported", K(ret), K(arg_table_name)); + LOG_WARN("partitioned table not supported", K(ret), K(table_name)); } else { tablet_id = table_schema_->get_tablet_id(); } @@ -126,19 +166,24 @@ int ObTableCtx::init_common(ObTableApiCredential &credential, 0, /* expire_renew_time */ is_cache_hit, ls_id_))) { - LOG_WARN("fail to get ls id", K(ret), K(tablet_id), K(arg_table_name)); + LOG_WARN("fail to get ls id", K(ret), K(tablet_id), K(table_name)); } else if (!is_scan_ && OB_FAIL(adjust_entity())) { LOG_WARN("fail to adjust entity", K(ret)); } else { tenant_id_ = tenant_id; database_id_ = database_id; - table_name_ = arg_table_name; + table_name_ = table_name; ref_table_id_ = table_schema_->get_table_id(); index_table_id_ = ref_table_id_; tablet_id_ = tablet_id; index_tablet_id_ = tablet_id_; tenant_schema_version_ = tenant_schema->get_schema_version(); timeout_ts_ = timeout_ts; + is_ttl_table_ = !table_schema_->get_ttl_definition().empty(); + } + + if (OB_SUCC(ret) && OB_FAIL(init_phy_plan_ctx())) { + LOG_WARN("fail to init physical plan ctx", KR(ret)); } return ret; @@ -639,18 +684,12 @@ int ObTableCtx::init_scan(const ObTableQuery &query, int ObTableCtx::init_insert() { int ret = OB_SUCCESS; - void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); - } else { - ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); - phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 - phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); - exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); - if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { - LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); - } + ObPhysicalPlanCtx *phy_plan_ctx = get_physical_plan_ctx(); + if (OB_ISNULL(phy_plan_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("phy_plan_ctx is null", K(ret)); + } else if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { + LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); } return init_dml_related_tid(); } @@ -766,6 +805,23 @@ int ObTableCtx::init_delete() return ret; } +int ObTableCtx::init_phy_plan_ctx() +{ + int ret = OB_SUCCESS; + void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); + } else { + ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); + phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 + phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); + phy_plan_ctx->set_cur_time(ObTimeUtility::current_time()); + exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); + } + return ret; +} + int ObTableCtx::init_replace() { int ret = OB_SUCCESS; @@ -773,24 +829,47 @@ int ObTableCtx::init_replace() if (OB_FAIL(init_dml_related_tid())) { LOG_WARN("fail to init dml related tids", K(ret)); } else { - void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); - } else { - ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); - phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 - phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); - exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); - if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { - LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); - } + ObPhysicalPlanCtx *phy_plan_ctx = get_physical_plan_ctx(); + if (OB_ISNULL(phy_plan_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("phy_plan_ctx is null", K(ret)); + } else if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { + LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); } } return ret; } +int ObTableCtx::init_ttl_delete(ObRowkey &start_key) +{ + int ret = OB_SUCCESS; + ObTableQuery query; + ObNewRange range; + ObRowkey real_start_key; + range.end_key_.set_max_row(); + set_is_ttl_table(false); + if (!start_key.is_valid()) { + real_start_key.set_min_row(); + } else { + real_start_key = start_key; + } + + range.start_key_ = real_start_key; + if (OB_FAIL(query.add_scan_range(range))) { + LOG_WARN("fail to generate key ranges", KR(ret), K(range)); + } else if (OB_FAIL(init_scan(query, false/* is_weak_read */))) { + LOG_WARN("fail to init scan ctx", KR(ret), K(query)); + } + + // 2. init related index table id + if (OB_SUCC(ret) && OB_FAIL(init_dml_related_tid())) { + LOG_WARN("fail to init dml related table ids", K(ret)); + } + + return ret; +} + int ObTableCtx::init_insert_up() { int ret = OB_SUCCESS; @@ -799,18 +878,12 @@ int ObTableCtx::init_insert_up() if (OB_FAIL(init_update())) { LOG_WARN("fail to init update", K(ret)); } else { - void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); - } else { - ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); - phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 - phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); - exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); - if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { - LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); - } + ObPhysicalPlanCtx *phy_plan_ctx = get_physical_plan_ctx(); + if (OB_ISNULL(phy_plan_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("phy_plan_ctx is null", K(ret)); + } else if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { + LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); } } diff --git a/src/observer/table/ob_table_context.h b/src/observer/table/ob_table_context.h index 75add83ea1..f0f35f5493 100644 --- a/src/observer/table/ob_table_context.h +++ b/src/observer/table/ob_table_context.h @@ -53,8 +53,9 @@ enum ObTableExecutorType TABLE_API_EXEC_INSERT_UP = 5, TABLE_API_EXEC_REPLACE = 6, TABLE_API_EXEC_LOCK = 7, + TABLE_API_EXEC_TTL = 8, // append new executor type here - TABLE_API_EXEC_MAX = 8 + TABLE_API_EXEC_MAX }; // 1.用于存放整个process过程中需要的通用上下文信息 @@ -119,6 +120,7 @@ public: return_affected_entity_ = false; return_rowkey_ = false; cur_cluster_version_ = GET_MIN_CLUSTER_VERSION(); + is_ttl_table_ = false; } virtual ~ObTableCtx() {} @@ -148,7 +150,8 @@ public: // insert up to string K_(is_for_insertup), K_(entity_type), - K_(cur_cluster_version)); + K_(cur_cluster_version), + K_(is_ttl_table)); public: //////////////////////////////////////// getter //////////////////////////////////////////////// // for common @@ -184,6 +187,8 @@ public: OB_INLINE bool is_get() const { return is_get_; } OB_INLINE bool is_read_latest() const { return read_latest_; } OB_INLINE common::ObQueryFlag::ScanOrder get_scan_order() const { return scan_order_; } + OB_INLINE ObIArray& get_filter_exprs() { return filter_exprs_; } + OB_INLINE const ObIArray& get_filter_exprs() const { return filter_exprs_; } OB_INLINE const ObIArray& get_select_exprs() const { return select_exprs_; } OB_INLINE const ObIArray& get_rowkey_exprs() const { return rowkey_exprs_; } OB_INLINE const ObIArray& get_index_exprs() const { return index_exprs_; } @@ -213,6 +218,10 @@ public: OB_INLINE const ObITableEntity* get_entity() const { return entity_; } OB_INLINE ObTableEntityType get_entity_type() const { return entity_type_; } OB_INLINE bool is_htable() const { return ObTableEntityType::ET_HKV == entity_type_; } + OB_INLINE bool is_insert() const + { + return ObTableOperationType::Type::INSERT == operation_type_; + } // for htable OB_INLINE const ObTableBatchOperation* get_batch_operation() const { return batch_op_; } // for increment/append @@ -264,11 +273,17 @@ public: && operation_type_ != ObTableOperationType::SCAN; } public: - // 初始化common部分(不包括expr_info_, exec_ctx_, all_exprs_) + // 基于 table name 初始化common部分(不包括expr_info_, exec_ctx_) int init_common(ObTableApiCredential &credential, const common::ObTabletID &arg_tablet_id, const common::ObString &arg_table_name, const int64_t &timeout_ts); + + // 基于 table id 初始化common部分(不包括expr_info_, exec_ctx_) + int init_common(ObTableApiCredential &credential, + const common::ObTabletID &arg_tablet_id, + const uint64_t table_id, + const int64_t &timeout_ts); // 初始化 insert 相关 int init_insert(); // 初始化scan相关(不包括表达分类) @@ -298,6 +313,12 @@ public: int init_das_context(ObDASCtx &das_ctx); // 更新全局自增值 int update_auto_inc_value(); + // init table context for ttl operation + bool is_ttl_table() const { return is_ttl_table_; } + + void set_is_ttl_table(bool is_ttl_table) { is_ttl_table_ = is_ttl_table; } + int init_phy_plan_ctx(); + int init_ttl_delete(ObRowkey &start_key); public: // convert lob的allocator需要保证obj写入表达式后才能析构 static int convert_lob(common::ObIAllocator &allocator, ObObj &obj); @@ -351,6 +372,13 @@ private: // 获取索引表的tablet_id int get_related_tablet_id(const share::schema::ObTableSchema &index_schema, common::ObTabletID &related_tablet_id); + + + // 初始化 table schema 之后的 common 部分 + int inner_init_common(ObTableApiCredential &credential, + const common::ObTabletID &arg_tablet_id, + const common::ObString &table_name, + const int64_t &timeout_ts); private: bool is_init_; common::ObIAllocator &allocator_; // processor allocator @@ -384,6 +412,7 @@ private: common::ObArray select_exprs_; common::ObArray rowkey_exprs_; common::ObArray index_exprs_; + common::ObArray filter_exprs_; common::ObArray select_col_ids_; // 基于schema序的select column id common::ObArray query_col_ids_; // 用户查询的select column id common::ObArray query_col_names_; // 用户查询的select column name，引用的是schema上的列名 @@ -421,6 +450,7 @@ private: uint64_t cur_cluster_version_; // for rowkey constraint info common::ObSEArray all_column_ref_exprs_; + bool is_ttl_table_; private: DISALLOW_COPY_AND_ASSIGN(ObTableCtx); }; @@ -453,6 +483,7 @@ public: lookup_ctdef_(nullptr), lookup_loc_meta_(nullptr), output_exprs_(allocator), + filter_exprs_(allocator), allocator_(allocator) { } @@ -464,6 +495,7 @@ public: sql::ObDASTableLocMeta *lookup_loc_meta_; ExprFixedArray output_exprs_; + ExprFixedArray filter_exprs_; common::ObIAllocator &allocator_; }; @@ -709,6 +741,43 @@ public: ObDASLockRtDef das_rtdef_; }; +struct ObTableTTLCtDef +{ +public: + ObTableTTLCtDef(common::ObIAllocator &alloc) + : ins_ctdef_(alloc), + del_ctdef_(alloc), + upd_ctdef_(alloc), + expire_expr_(nullptr), + alloc_(alloc) + { + } + TO_STRING_KV(K_(ins_ctdef), + K_(del_ctdef)); + ObTableInsCtDef ins_ctdef_; + ObTableDelCtDef del_ctdef_; + ObTableUpdCtDef upd_ctdef_; + ObExpr *expire_expr_; + common::ObIAllocator &alloc_; +}; + +struct ObTableTTLRtDef +{ +public: + ObTableTTLRtDef() + : ins_rtdef_(), + del_rtdef_(), + upd_rtdef_() + { + } + TO_STRING_KV(K_(ins_rtdef), + K_(del_rtdef), + K_(upd_rtdef)) + ObTableInsRtDef ins_rtdef_; + ObTableDelRtDef del_rtdef_; + ObTableUpdRtDef upd_rtdef_; +}; + } // end namespace table } // end namespace oceanbase diff --git a/src/observer/table/ob_table_end_trans_cb.cpp b/src/observer/table/ob_table_end_trans_cb.cpp index 082d445352..0e875203a5 100644 --- a/src/observer/table/ob_table_end_trans_cb.cpp +++ b/src/observer/table/ob_table_end_trans_cb.cpp @@ -17,7 +17,8 @@ using namespace oceanbase::common; using namespace oceanbase::table; ObTableAPITransCb::ObTableAPITransCb() :tx_desc_(NULL), - ref_count_(2) + ref_count_(2), + lock_handle_(nullptr) {} ObTableAPITransCb::~ObTableAPITransCb() @@ -35,6 +36,11 @@ void ObTableAPITransCb::destroy_cb_if_no_ref() } } +void ObTableAPITransCb::set_lock_handle(ObHTableLockHandle *lock_handle) +{ + lock_handle_ = lock_handle; +} + //////////////////////////////////////////////////////////////// void ObTableExecuteEndTransCb::callback(int cb_param) { @@ -49,6 +55,9 @@ void ObTableExecuteEndTransCb::callback(int cb_param) MTL(transaction::ObTransService*)->release_tx(*tx_desc_); tx_desc_ = NULL; } + if (lock_handle_ != nullptr) { + HTABLE_LOCK_MGR->release_handle(*lock_handle_); + } this->handin(); CHECK_BALANCE("[table async callback]"); if (cb_param != OB_SUCCESS) { @@ -62,6 +71,7 @@ void ObTableExecuteEndTransCb::callback(int cb_param) } else { LOG_INFO("async send execute response", K(cb_param)); } + this->destroy_cb_if_no_ref(); } @@ -102,6 +112,9 @@ void ObTableBatchExecuteEndTransCb::callback(int cb_param) MTL(transaction::ObTransService*)->release_tx(*tx_desc_); tx_desc_ = NULL; } + if (lock_handle_ != nullptr) { + HTABLE_LOCK_MGR->release_handle(*lock_handle_); + } this->handin(); CHECK_BALANCE("[table batch async callback]"); if (cb_param != OB_SUCCESS) { diff --git a/src/observer/table/ob_table_end_trans_cb.h b/src/observer/table/ob_table_end_trans_cb.h index 2adc15623b..d226c2895d 100644 --- a/src/observer/table/ob_table_end_trans_cb.h +++ b/src/observer/table/ob_table_end_trans_cb.h @@ -15,6 +15,7 @@ #include "ob_rpc_async_response.h" #include "sql/ob_end_trans_callback.h" #include "share/table/ob_table.h" +#include "ob_htable_lock_mgr.h" namespace oceanbase { namespace table @@ -26,12 +27,15 @@ public: virtual ~ObTableAPITransCb(); void destroy_cb_if_no_ref(); void set_tx_desc(transaction::ObTxDesc *tx_desc) { tx_desc_ = tx_desc; } + void set_lock_handle(ObHTableLockHandle *lock_handle); protected: transaction::ObTxDesc *tx_desc_; private: int32_t ref_count_; // disallow copy DISALLOW_COPY_AND_ASSIGN(ObTableAPITransCb); +protected: + ObHTableLockHandle *lock_handle_; // hbase row lock handle }; diff --git a/src/observer/table/ob_table_execute_processor.cpp b/src/observer/table/ob_table_execute_processor.cpp index 0c2fa65ea2..bd74682b27 100644 --- a/src/observer/table/ob_table_execute_processor.cpp +++ b/src/observer/table/ob_table_execute_processor.cpp @@ -123,8 +123,14 @@ int ObTableApiExecuteP::init_tb_ctx() } else { switch(op_type) { case ObTableOperationType::INSERT: { - if (OB_FAIL(tb_ctx_.init_insert())) { - LOG_WARN("fail to init insert ctx", K(ret), K(tb_ctx_)); + if (tb_ctx_.is_ttl_table()) { + if (OB_FAIL(tb_ctx_.init_insert_up())) { + LOG_WARN("fail to init insert up ctx", K(ret), K(tb_ctx_)); + } + } else { + if (OB_FAIL(tb_ctx_.init_insert())) { + LOG_WARN("fail to init insert ctx", K(ret), K(tb_ctx_)); + } } break; } @@ -211,11 +217,19 @@ int ObTableApiExecuteP::try_process() } else if (OB_UNLIKELY(!is_index_supported)) { ret = OB_NOT_SUPPORTED; LOG_WARN("index type is not supported by table api", K(ret)); + } else if (OB_FAIL(check_arg2())) { + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_tb_ctx())) { + LOG_WARN("fail to init tb ctx", K(ret)); } else { switch (table_operation.type()) { case ObTableOperationType::INSERT: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_INSERT; - ret = process_dml_op(); + if (tb_ctx_.is_ttl_table()) { + ret = process_dml_op(); + } else { + ret = process_dml_op(); + } break; case ObTableOperationType::GET: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_GET; @@ -231,7 +245,11 @@ int ObTableApiExecuteP::try_process() break; case ObTableOperationType::INSERT_OR_UPDATE: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_INSERT_OR_UPDATE; - ret = process_dml_op(); + if (tb_ctx_.is_ttl_table()) { + ret = process_dml_op(); + } else { + ret = process_dml_op(); + } break; case ObTableOperationType::REPLACE: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_REPLACE; @@ -239,11 +257,19 @@ int ObTableApiExecuteP::try_process() break; case ObTableOperationType::INCREMENT: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_INCREMENT; - ret = process_dml_op(); + if (tb_ctx_.is_ttl_table()) { + ret = process_dml_op(); + } else { + ret = process_dml_op(); + } break; case ObTableOperationType::APPEND: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_APPEND; - ret = process_dml_op(); + if (tb_ctx_.is_ttl_table()) { + ret = process_dml_op(); + } else { + ret = process_dml_op(); + } break; default: ret = OB_INVALID_ARGUMENT; @@ -253,6 +279,12 @@ int ObTableApiExecuteP::try_process() audit_row_count_ = 1; } + if (OB_FAIL(ret)) { + // init_tb_ctx will return some replaceable error code + result_.set_errno(ret); + table::ObTableApiUtil::replace_ret_code(ret); + } + #ifndef NDEBUG // debug mode LOG_INFO("[TABLE] execute operation", K(ret), K_(arg), K_(result), "timeout", rpc_pkt_->get_timeout(), K_(retry_count)); diff --git a/src/observer/table/ob_table_execute_processor.h b/src/observer/table/ob_table_execute_processor.h index 2bf0e30d21..8b1f6883e3 100644 --- a/src/observer/table/ob_table_execute_processor.h +++ b/src/observer/table/ob_table_execute_processor.h @@ -55,16 +55,12 @@ private: int process_dml_op() { int ret = OB_SUCCESS; - if (OB_FAIL(check_arg2())) { - SERVER_LOG(WARN, "fail to check arg", K(ret)); - } else if (OB_FAIL(init_tb_ctx())) { - SERVER_LOG(WARN, "fail to init tb ctx", K(ret)); - } else if (OB_FAIL(start_trans(false, /* is_readonly */ - sql::stmt::T_INSERT, - arg_.consistency_level_, - tb_ctx_.get_table_id(), - tb_ctx_.get_ls_id(), - get_timeout_ts()))) { + if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { SERVER_LOG(WARN, "fail to start trans", K(ret)); } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { SERVER_LOG(WARN, "fail to init trans", K(ret)); diff --git a/src/observer/table/ob_table_executor.h b/src/observer/table/ob_table_executor.h index 3e68d2498a..e10a24ae43 100644 --- a/src/observer/table/ob_table_executor.h +++ b/src/observer/table/ob_table_executor.h @@ -50,7 +50,7 @@ public: const ObTableApiExecutor* get_child() const { return child_; } OB_INLINE const ObTableCtx& get_table_ctx() const { return tb_ctx_; } OB_INLINE sql::ObEvalCtx &get_eval_ctx() { return eval_ctx_; } - void clear_evaluated_flag(); + virtual void clear_evaluated_flag(); protected: ObTableCtx &tb_ctx_; diff --git a/src/observer/table/ob_table_executor_reg.h b/src/observer/table/ob_table_executor_reg.h index da2de12cb4..83a77ed70c 100644 --- a/src/observer/table/ob_table_executor_reg.h +++ b/src/observer/table/ob_table_executor_reg.h @@ -19,6 +19,7 @@ #include "ob_table_insert_up_executor.h" #include "ob_table_replace_executor.h" #include "ob_table_lock_executor.h" +#include "ttl/ob_table_ttl_executor.h" namespace oceanbase { @@ -61,6 +62,7 @@ REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_UPDATE, ObTableApiUpdateSpec, ObTable REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_INSERT_UP, ObTableApiInsertUpSpec, ObTableApiInsertUpExecutor); REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_REPLACE, ObTableApiReplaceSpec, ObTableApiReplaceExecutor); REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_LOCK, ObTableApiLockSpec, ObTableApiLockExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_TTL, ObTableApiTTLSpec, ObTableApiTTLExecutor); #undef REGISTER_TABLE_API_EXECUTOR diff --git a/src/observer/table/ob_table_insert_up_executor.cpp b/src/observer/table/ob_table_insert_up_executor.cpp index 182d41651c..16ffe80c7e 100644 --- a/src/observer/table/ob_table_insert_up_executor.cpp +++ b/src/observer/table/ob_table_insert_up_executor.cpp @@ -40,32 +40,6 @@ int ObTableApiInsertUpExecutor::generate_insert_up_rtdef(const ObTableInsUpdCtDe return ret; } -int ObTableApiInsertUpExecutor::modify_htable_timestamp() -{ - int ret = OB_SUCCESS; - int64_t now_ms = -ObHTableUtils::current_time_millis(); - const ObITableEntity *entity = static_cast(tb_ctx_.get_entity()); - if (entity->get_rowkey_size() != 3) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("htable should be with 3 rowkey columns", K(ret), K(entity)); - } else { - ObRowkey rowkey = entity->get_rowkey(); - ObObj &t_obj = const_cast(rowkey.get_obj_ptr()[ObHTableConstants::COL_IDX_T]); // column T - ObHTableCellEntity3 htable_cell(entity); - bool row_is_null = htable_cell.last_get_is_null(); - int64_t timestamp = htable_cell.get_timestamp(); - bool timestamp_is_null = htable_cell.last_get_is_null(); - if (row_is_null || timestamp_is_null) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument for htable put", K(ret), K(row_is_null), K(timestamp_is_null)); - } else if (ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // update timestamp iff LATEST_TIMESTAMP - t_obj.set_int(now_ms); - } - } - - return ret; -} - int ObTableApiInsertUpExecutor::open() { int ret = OB_SUCCESS; @@ -107,14 +81,6 @@ int ObTableApiInsertUpExecutor::open() return ret; } -void ObTableApiInsertUpExecutor::set_need_fetch_conflict() -{ - insert_up_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = true; - dml_rtctx_.set_non_sub_full_task(); - upd_rtctx_.set_pick_del_task_first(); - upd_rtctx_.set_non_sub_full_task(); -} - int ObTableApiInsertUpExecutor::refresh_exprs_frame(const ObTableEntity *entity) { int ret = OB_SUCCESS; @@ -149,10 +115,9 @@ int ObTableApiInsertUpExecutor::get_next_row_from_child() return ret; } -int ObTableApiInsertUpExecutor::try_insert_row(bool &is_iter_end, int64_t &insert_rows) +int ObTableApiInsertUpExecutor::try_insert_row() { int ret = OB_SUCCESS; - is_iter_end = false; const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); while (OB_SUCC(ret)) { @@ -164,16 +129,11 @@ int ObTableApiInsertUpExecutor::try_insert_row(bool &is_iter_end, int64_t &inser LOG_WARN("fail to insert row to das", K(ret)); } else { cur_idx_++; - insert_rows++; } } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - is_iter_end = true; - if (OB_FAIL(execute_das_task(dml_rtctx_, false))) { - LOG_WARN("fail to execute all das task", K(ret)); - } + if (OB_ITER_END == ret && OB_FAIL(execute_das_task(dml_rtctx_, false/* del_task_ahead */))) { + LOG_WARN("fail to execute das task"); } return ret; @@ -198,92 +158,6 @@ int ObTableApiInsertUpExecutor::try_update_row() return ret; } -int ObTableApiInsertUpExecutor::execute_das_task(ObDMLRtCtx &dml_rtctx, - bool del_task_ahead) -{ - int ret = OB_SUCCESS; - - if (dml_rtctx.das_ref_.has_task()) { - if (del_task_ahead) { - if (OB_FAIL(dml_rtctx.das_ref_.pick_del_task_to_first())) { - LOG_WARN("fail to remove delete das task first", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) { - LOG_WARN("fail to execute all das task", K(ret)); - } - } - } - - return ret; -} - -int ObTableApiInsertUpExecutor::fetch_conflict_rowkey() -{ - int ret = OB_SUCCESS; - DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); - - while (OB_SUCC(ret) && !task_iter.is_end()) { - if (OB_FAIL(get_next_conflict_rowkey(task_iter, conflict_checker_))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next conflict rowkey from das_result", K(ret)); - } - } else if (OB_FAIL(conflict_checker_.build_primary_table_lookup_das_task())) { - LOG_WARN("fail to build lookup_das_task", K(ret)); - } - } - - ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); - return ret; -} - -int ObTableApiInsertUpExecutor::reset_das_env() -{ - int ret = OB_SUCCESS; - // 释放第一次try insert的das task - if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { - LOG_WARN("fail to close all das task", K(ret)); - } else { - dml_rtctx_.das_ref_.reuse(); - } - - // 因为第二次插入不需要fetch conflict result了 - insert_up_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = false; - insert_up_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_ = false; - - return ret; -} - -int ObTableApiInsertUpExecutor::check_whether_row_change(const ObChunkDatumStore::StoredRow &upd_old_row, - const ObChunkDatumStore::StoredRow &upd_new_row) -{ - int ret = OB_SUCCESS; - - if (tb_ctx_.is_inc_or_append()) { - is_row_changed_ = true; - } else if (lib::is_mysql_mode()) { - const ObTableUpdCtDef &upd_ctdef = insert_up_spec_.get_ctdef().upd_ctdef_; - const ObExprPtrIArray &old_row = upd_ctdef.old_row_; - const ObExprPtrIArray &new_row = upd_ctdef.new_row_; - FOREACH_CNT_X(info, upd_ctdef.assign_columns_, OB_SUCC(ret) && !is_row_changed_) { - const uint64_t idx = info->projector_index_; - if (idx >= upd_old_row.cnt_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid assign idx", K(ret), K(idx), K(upd_old_row.cnt_)); - } else { - is_row_changed_ = !ObDatum::binary_equal(upd_old_row.cells()[idx], upd_new_row.cells()[idx]); - } - } - } else { - //in oracle mode, no matter whether the updated row is changed or not, - //the row will be updated in the storage - is_row_changed_ = true; - } - - return ret; -} - // 通过主键在conflict_checker_中找到冲突旧行，执行更新 // 注意，这里更新后还可能出现二级索引冲突，eg: // create table t (C1 int, C2 varchar(10), primary key(C1), UNIQUE KEY idx_c2 (C2)); @@ -322,7 +196,10 @@ int ObTableApiInsertUpExecutor::do_insert_up_cache() if (OB_ISNULL(upd_old_row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("upd_old_row is NULL", K(ret)); - } else if (OB_FAIL(check_whether_row_change(*upd_old_row, *upd_new_row))) { + } else if (OB_FAIL(check_whether_row_change(*upd_old_row, + *upd_new_row, + insert_up_spec_.get_ctdef().upd_ctdef_, + is_row_changed_))) { LOG_WARN("fail to check whether row change", K(ret)); } else if (is_row_changed_) { // do update @@ -363,193 +240,6 @@ int ObTableApiInsertUpExecutor::prepare_final_insert_up_task() return ret; } -int ObTableApiInsertUpExecutor::delete_upd_old_row_to_das(const ObRowkey &constraint_rowkey, - const sql::ObConflictValue &constraint_value) -{ - int ret = OB_SUCCESS; - ObDASTabletLoc *tablet_loc = nullptr; - - if (OB_FAIL(calc_tablet_loc(tablet_loc))) { - LOG_WARN("fail to calc tablet location", K(ret)); - } else { - const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); - const ObTableUpdCtDef &upd_ctdef = insert_up_ctdef.upd_ctdef_; - ObTableUpdRtDef &upd_rtdef = insert_up_rtdef_.upd_rtdef_; - if (OB_ISNULL(upd_ctdef.ddel_ctdef_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddel_ctdef can't be null", K(ret)); - } else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) { - if (OB_FAIL(ObDASTaskFactory::alloc_das_rtdef(DAS_OP_TABLE_DELETE, - tb_ctx_.get_allocator(), - upd_rtdef.ddel_rtdef_))) { - LOG_WARN("fail to create das delete rtdef", K(ret)); - } else if (OB_FAIL(generate_del_rtdef_for_update(upd_ctdef, upd_rtdef))) { - LOG_WARN("fail to generate del rtdef for update", K(ret), K(upd_ctdef), K(upd_rtdef)); - } - } - ObChunkDatumStore::StoredRow* stored_row = nullptr; - if (OB_FAIL(ret)) { - //do nothing - } else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ddel_rtdef is null", K(ret)); - } else if (OB_FAIL(ObDMLService::delete_row(*upd_ctdef.ddel_ctdef_, - *upd_rtdef.ddel_rtdef_, - tablet_loc, - upd_rtctx_, - upd_ctdef.old_row_, - stored_row))) { - LOG_WARN("fail to delete row with das", K(ret)); - } - } - - return ret; -} - -int ObTableApiInsertUpExecutor::generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, - ObTableUpdRtDef &upd_rtdef) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(init_das_dml_rtdef(*upd_ctdef.ddel_ctdef_, - *upd_rtdef.ddel_rtdef_, - nullptr))) { - LOG_WARN("fail to init das dml rtdef", K(ret)); - } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_del_ctdefs_, - upd_rtdef.related_del_rtdefs_))) { - LOG_WARN("fail to init related das ctdef", K(ret)); - } else { - upd_rtdef.ddel_rtdef_->related_ctdefs_ = &upd_ctdef.related_del_ctdefs_; - upd_rtdef.ddel_rtdef_->related_rtdefs_ = &upd_rtdef.related_del_rtdefs_; - } - - return ret; -} - -int ObTableApiInsertUpExecutor::generate_ins_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, - ObTableUpdRtDef &upd_rtdef) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(init_das_dml_rtdef(*upd_ctdef.dins_ctdef_, - *upd_rtdef.dins_rtdef_, - nullptr))) { - LOG_WARN("fail to init das dml rtdef", K(ret)); - } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_ins_ctdefs_, - upd_rtdef.related_ins_rtdefs_))) { - LOG_WARN("fail to init related das ctdef", K(ret)); - } else { - upd_rtdef.dins_rtdef_->related_ctdefs_ = &upd_ctdef.related_ins_ctdefs_; - upd_rtdef.dins_rtdef_->related_rtdefs_ = &upd_rtdef.related_ins_rtdefs_; - } - - return ret; -} - -int ObTableApiInsertUpExecutor::insert_upd_new_row_to_das() -{ - int ret = OB_SUCCESS; - const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); - ObDASTabletLoc *tablet_loc = nullptr; - - if (OB_FAIL(calc_tablet_loc(tablet_loc))) { - LOG_WARN("fail to calc tablet location", K(ret)); - } else { - const ObTableUpdCtDef &upd_ctdef = insert_up_ctdef.upd_ctdef_; - ObTableUpdRtDef &upd_rtdef = insert_up_rtdef_.upd_rtdef_; - if (OB_ISNULL(upd_ctdef.dins_ctdef_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("dins_ctdef_ can't be null", K(ret)); - } else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) { - if (OB_FAIL(ObDASTaskFactory::alloc_das_rtdef(DAS_OP_TABLE_INSERT, - tb_ctx_.get_allocator(), - upd_rtdef.dins_rtdef_))) { - LOG_WARN("fail to create das insert rtdef", K(ret)); - } else if (OB_FAIL(generate_ins_rtdef_for_update(upd_ctdef, upd_rtdef))) { - LOG_WARN("fail to generate del rtdef for update", K(ret), K(upd_ctdef), K(upd_rtdef)); - } - } - - clear_evaluated_flag(); - ObChunkDatumStore::StoredRow* stored_row = nullptr; - if (OB_FAIL(ret)) { - //do nothing - } else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("dins_rtdef_ is null", K(ret)); - } else if (OB_FAIL(ObDMLService::insert_row(*upd_ctdef.dins_ctdef_, - *upd_rtdef.dins_rtdef_, - tablet_loc, - upd_rtctx_, - upd_ctdef.new_row_, - stored_row))) { - LOG_WARN("fail to insert row with das", K(ret)); - } - } - - return ret; -} - -int ObTableApiInsertUpExecutor::to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, - const ObRowkey &constraint_rowkey) -{ - int ret = OB_SUCCESS; - const ObTableSchema *table_schema = tb_ctx_.get_table_schema(); - const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); - const ObTableUpdCtDef &upd_ctdef = insert_up_ctdef.upd_ctdef_; - const ObIArray &new_row = upd_ctdef.new_row_; - if (OB_UNLIKELY(store_row.cnt_ != new_row.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("datum count mismatch", K(ret), K(store_row.cnt_), K(new_row.count())); - } else { - // 1. refresh rowkey expr datum - const int64_t rowkey_col_cnt = tb_ctx_.get_table_schema()->get_rowkey_column_num(); - for (uint64_t i = 0; OB_SUCC(ret) && i < rowkey_col_cnt; ++i) { - const ObExpr *expr = new_row.at(i); - expr->locate_expr_datum(eval_ctx_) = store_row.cells()[i]; - expr->get_eval_info(eval_ctx_).evaluated_ = true; - expr->get_eval_info(eval_ctx_).projected_ = true; - } - - // 2. refresh assign column expr datum - const ObTableCtx::ObAssignIds &assign_ids = tb_ctx_.get_assign_ids(); - const int64_t N = assign_ids.count(); - for (uint64_t i = 0; OB_SUCC(ret) && i < N; ++i) { - uint64_t assign_id = assign_ids.at(i).idx_; - const ObColumnSchemaV2 *col_schema = nullptr; - if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(assign_id))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(assign_id), K(*table_schema)); - } else if (assign_id >= store_row.cnt_) { - ret = OB_ERROR_OUT_OF_RANGE; - LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(store_row.cnt_)); - } else if (assign_id >= new_row.count()) { - ret = OB_ERROR_OUT_OF_RANGE; - LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(new_row.count())); - } else if (col_schema->is_virtual_generated_column()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("should not have virtual generated expr", K(ret)); - } else if (col_schema->is_stored_generated_column()) { - ObTableCtx &ctx = const_cast(tb_ctx_); - if (OB_FAIL(ObTableExprCgService::refresh_generated_column_related_frame(ctx, - upd_ctdef.old_row_, - upd_ctdef.full_assign_row_, - assign_ids, - *col_schema))) { - LOG_WARN("fail to refresh generated column related frame", K(ret), K(ctx), K(*col_schema)); - } - } else { - const ObExpr *expr = new_row.at(assign_id); - expr->locate_expr_datum(eval_ctx_) = store_row.cells()[assign_id]; - expr->get_eval_info(eval_ctx_).evaluated_ = true; - expr->get_eval_info(eval_ctx_).projected_ = true; - } - } - } - - return ret; -} - int ObTableApiInsertUpExecutor::do_update(const ObRowkey &constraint_rowkey, const ObConflictValue &constraint_value) { @@ -562,14 +252,26 @@ int ObTableApiInsertUpExecutor::do_update(const ObRowkey &constraint_rowkey, // base_line 和 curr_row 都存在 OZ(constraint_value.baseline_datum_row_->to_expr(get_primary_table_upd_old_row(), eval_ctx_)); - OZ(delete_upd_old_row_to_das(constraint_rowkey, constraint_value)); - OZ(to_expr_skip_old(*constraint_value.current_datum_row_, constraint_rowkey)); + OZ(delete_upd_old_row_to_das(constraint_rowkey, + constraint_value, + insert_up_spec_.get_ctdef().upd_ctdef_, + insert_up_rtdef_.upd_rtdef_, + upd_rtctx_)); + OZ(to_expr_skip_old(*constraint_value.current_datum_row_, + constraint_rowkey, + insert_up_spec_.get_ctdef().upd_ctdef_)); clear_evaluated_flag(); - OZ(insert_upd_new_row_to_das()); + OZ(insert_upd_new_row_to_das(insert_up_spec_.get_ctdef().upd_ctdef_, + insert_up_rtdef_.upd_rtdef_, + upd_rtctx_)); } else if (NULL == constraint_value.baseline_datum_row_ && - NULL != constraint_value.current_datum_row_) { - OZ(to_expr_skip_old(*constraint_value.current_datum_row_, constraint_rowkey)); - OZ(insert_upd_new_row_to_das()); + NULL != constraint_value.current_datum_row_) { // 单单是唯一索引冲突的时候，会走这个分支 + OZ(to_expr_skip_old(*constraint_value.current_datum_row_, + constraint_rowkey, + insert_up_spec_.get_ctdef().upd_ctdef_)); + OZ(insert_upd_new_row_to_das(insert_up_spec_.get_ctdef().upd_ctdef_, + insert_up_rtdef_.upd_rtdef_, + upd_rtctx_)); } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected constraint_value", K(ret)); @@ -582,31 +284,27 @@ int ObTableApiInsertUpExecutor::do_update(const ObRowkey &constraint_rowkey, int ObTableApiInsertUpExecutor::get_next_row() { int ret = OB_SUCCESS; - bool is_iter_end = false; + transaction::ObTxSEQ savepoint_no; - while (OB_SUCC(ret) && !is_iter_end) { - int64_t insert_rows = 0; - transaction::ObTxSEQ savepoint_no; - // must set conflict_row fetch flag - set_need_fetch_conflict(); - if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(exec_ctx_, savepoint_no))) { - LOG_WARN("fail to create save_point", K(ret)); - } else if (OB_FAIL(try_insert_row(is_iter_end, insert_rows))) { - LOG_WARN("fail to try insert row", K(ret)); - } else if (!is_duplicated()) { - insert_rows_ += insert_rows; - LOG_TRACE("try insert is not duplicated", K(ret), K(insert_rows_)); - } else if (OB_FAIL(fetch_conflict_rowkey())) { - LOG_WARN("fail to fetch conflict row", K(ret)); - } else if (OB_FAIL(reset_das_env())) { - // 这里需要reuse das 相关信息 - LOG_WARN("fail to reset das env", K(ret)); - } else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(exec_ctx_, savepoint_no))) { - // 本次插入存在冲突, 回滚到save_point - LOG_WARN("fail to rollback to save_point", K(ret)); - } else if (OB_FAIL(try_update_row())) { - LOG_WARN("fail to try update row", K(ret)); - } + // must set conflict_row fetch flag + set_need_fetch_conflict(upd_rtctx_, insert_up_rtdef_.ins_rtdef_); + if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(exec_ctx_, savepoint_no))) { + LOG_WARN("fail to create save_point", K(ret)); + } else if (OB_FAIL(try_insert_row())) { + LOG_WARN("fail to try insert row", K(ret)); + } else if (!is_duplicated()) { + insert_rows_ = 1; + LOG_TRACE("try insert is not duplicated", K(ret), K(insert_rows_)); + } else if (OB_FAIL(fetch_conflict_rowkey(conflict_checker_))) { + LOG_WARN("fail to fetch conflict row", K(ret)); + } else if (OB_FAIL(reset_das_env(insert_up_rtdef_.ins_rtdef_))) { + // 这里需要reuse das 相关信息 + LOG_WARN("fail to reset das env", K(ret)); + } else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(exec_ctx_, savepoint_no))) { + // 本次插入存在冲突, 回滚到save_point + LOG_WARN("fail to rollback to save_point", K(ret)); + } else if (OB_FAIL(try_update_row())) { + LOG_WARN("fail to try update row", K(ret)); } if (OB_SUCC(ret)) { diff --git a/src/observer/table/ob_table_insert_up_executor.h b/src/observer/table/ob_table_insert_up_executor.h index 8ad2967c8f..d4d0865d05 100644 --- a/src/observer/table/ob_table_insert_up_executor.h +++ b/src/observer/table/ob_table_insert_up_executor.h @@ -105,35 +105,16 @@ private: } int generate_insert_up_rtdef(const ObTableInsUpdCtDef &ctdef, ObTableInsUpdRtDef &rtdef); - void set_need_fetch_conflict(); int refresh_exprs_frame(const ObTableEntity *entity); int get_next_row_from_child(); - int try_insert_row(bool &is_iter_end, int64_t &insert_rows); + int try_insert_row(); int try_update_row(); - int execute_das_task(sql::ObDMLRtCtx &dml_rtctx, bool del_task_ahead); - int fetch_conflict_rowkey(); - int reset_das_env(); int do_insert_up_cache(); int prepare_final_insert_up_task(); int do_update(const ObRowkey &constraint_rowkey, const sql::ObConflictValue &constraint_value); int do_insert(const ObRowkey &constraint_rowkey, const sql::ObConflictValue &constraint_value); - int delete_upd_old_row_to_das(const ObRowkey &constraint_rowkey, - const sql::ObConflictValue &constraint_value); - int insert_upd_new_row_to_das(); - int generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, - ObTableUpdRtDef &upd_rtdef); - int generate_ins_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, - ObTableUpdRtDef &upd_rtdef); - int to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, - const ObRowkey &constraint_rowkey); - // for htable - int modify_htable_timestamp(); - - int check_whether_row_change(const ObChunkDatumStore::StoredRow &upd_old_row, - const ObChunkDatumStore::StoredRow &upd_new_row); - private: common::ObArenaAllocator allocator_; const ObTableApiInsertUpSpec &insert_up_spec_; diff --git a/src/observer/table/ob_table_modify_executor.cpp b/src/observer/table/ob_table_modify_executor.cpp index 5f0640e4b6..4359e9ed0e 100644 --- a/src/observer/table/ob_table_modify_executor.cpp +++ b/src/observer/table/ob_table_modify_executor.cpp @@ -14,6 +14,8 @@ #include "ob_table_modify_executor.h" #include "sql/engine/dml/ob_dml_service.h" #include "sql/das/ob_das_insert_op.h" +#include "ob_htable_utils.h" +#include "ob_table_cg_service.h" using namespace oceanbase::sql; @@ -315,5 +317,310 @@ int ObTableApiModifyExecutor::get_next_conflict_rowkey(DASTaskIter &task_iter, return ret; } +int ObTableApiModifyExecutor::modify_htable_timestamp() +{ + int ret = OB_SUCCESS; + int64_t now_ms = -ObHTableUtils::current_time_millis(); + const ObITableEntity *entity = static_cast(tb_ctx_.get_entity()); + if (entity->get_rowkey_size() != 3) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("htable should be with 3 rowkey columns", K(ret), K(entity)); + } else { + ObRowkey rowkey = entity->get_rowkey(); + ObObj &t_obj = const_cast(rowkey.get_obj_ptr()[ObHTableConstants::COL_IDX_T]); // column T + ObHTableCellEntity3 htable_cell(entity); + bool row_is_null = htable_cell.last_get_is_null(); + int64_t timestamp = htable_cell.get_timestamp(); + bool timestamp_is_null = htable_cell.last_get_is_null(); + if (row_is_null || timestamp_is_null) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument for htable put", K(ret), K(row_is_null), K(timestamp_is_null)); + } else if (ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // update timestamp iff LATEST_TIMESTAMP + t_obj.set_int(now_ms); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::fetch_conflict_rowkey(sql::ObConflictChecker &conflict_checker) +{ + int ret = OB_SUCCESS; + DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); + + while (OB_SUCC(ret) && !task_iter.is_end()) { + if (OB_FAIL(get_next_conflict_rowkey(task_iter, conflict_checker))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next conflict rowkey from das_result", K(ret)); + } + } else if (OB_FAIL(conflict_checker.build_primary_table_lookup_das_task())) { + LOG_WARN("fail to build lookup_das_task", K(ret)); + } + } + + ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); + return ret; +} + +int ObTableApiModifyExecutor::reset_das_env(ObTableInsRtDef &ins_rtdef) +{ + int ret = OB_SUCCESS; + + // 释放第一次try insert的das task + if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { + LOG_WARN("close all das task failed", K(ret)); + } else { + dml_rtctx_.das_ref_.reuse(); + // 第二次插入不需要fetch conflict result + ins_rtdef.das_rtdef_.need_fetch_conflict_ = false; + ins_rtdef.das_rtdef_.is_duplicated_ = false; + } + + return ret; +} + +int ObTableApiModifyExecutor::check_whether_row_change(const ObChunkDatumStore::StoredRow &upd_old_row, + const ObChunkDatumStore::StoredRow &upd_new_row, + const ObTableUpdCtDef &upd_ctdef, + bool &is_row_changed) +{ + int ret = OB_SUCCESS; + + if (tb_ctx_.is_inc_or_append()) { + is_row_changed = true; + } else if (lib::is_mysql_mode()) { + const ObExprPtrIArray &old_row = upd_ctdef.old_row_; + const ObExprPtrIArray &new_row = upd_ctdef.new_row_; + FOREACH_CNT_X(info, upd_ctdef.assign_columns_, OB_SUCC(ret) && !is_row_changed) { + const uint64_t idx = info->projector_index_; + if (idx >= upd_old_row.cnt_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid assign idx", K(ret), K(idx), K(upd_old_row.cnt_)); + } else { + is_row_changed = !ObDatum::binary_equal(upd_old_row.cells()[idx], upd_new_row.cells()[idx]); + } + } + } else { + //in oracle mode, no matter whether the updated row is changed or not, + //the row will be updated in the storage + is_row_changed = true; + } + + return ret; +} + +int ObTableApiModifyExecutor::to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, + const ObRowkey &constraint_rowkey, + const ObTableUpdCtDef &upd_ctdef) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = tb_ctx_.get_table_schema(); + const ObIArray &new_row = upd_ctdef.new_row_; + if (OB_UNLIKELY(store_row.cnt_ != new_row.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("datum count mismatch", K(ret), K(store_row.cnt_), K(new_row.count())); + } else { + // 1. refresh rowkey expr datum + const int64_t rowkey_col_cnt = tb_ctx_.get_table_schema()->get_rowkey_column_num(); + for (uint64_t i = 0; OB_SUCC(ret) && i < rowkey_col_cnt; ++i) { + const ObExpr *expr = new_row.at(i); + expr->locate_expr_datum(eval_ctx_) = store_row.cells()[i]; + expr->get_eval_info(eval_ctx_).evaluated_ = true; + expr->get_eval_info(eval_ctx_).projected_ = true; + } + + // 2. refresh assign column expr datum + const ObTableCtx::ObAssignIds &assign_ids = tb_ctx_.get_assign_ids(); + const int64_t N = assign_ids.count(); + for (uint64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + uint64_t assign_id = assign_ids.at(i).idx_; + const ObColumnSchemaV2 *col_schema = nullptr; + if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(assign_id))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(assign_id), K(*table_schema)); + } else if (assign_id >= store_row.cnt_) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(store_row.cnt_)); + } else if (assign_id >= new_row.count()) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(new_row.count())); + } else if (col_schema->is_virtual_generated_column()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should not have virtual generated expr", K(ret)); + } else if (col_schema->is_stored_generated_column()) { + ObTableCtx &ctx = const_cast(tb_ctx_); + if (OB_FAIL(ObTableExprCgService::refresh_generated_column_related_frame(ctx, + upd_ctdef.old_row_, + upd_ctdef.full_assign_row_, + assign_ids, + *col_schema))) { + LOG_WARN("fail to refresh generated column related frame", K(ret), K(ctx), K(*col_schema)); + } + } else { + const ObExpr *expr = new_row.at(assign_id); + expr->locate_expr_datum(eval_ctx_) = store_row.cells()[assign_id]; + expr->get_eval_info(eval_ctx_).evaluated_ = true; + expr->get_eval_info(eval_ctx_).projected_ = true; + } + } + } + + return ret; +} + +int ObTableApiModifyExecutor::generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(*upd_ctdef.ddel_ctdef_, + *upd_rtdef.ddel_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_del_ctdefs_, + upd_rtdef.related_del_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + upd_rtdef.ddel_rtdef_->related_ctdefs_ = &upd_ctdef.related_del_ctdefs_; + upd_rtdef.ddel_rtdef_->related_rtdefs_ = &upd_rtdef.related_del_rtdefs_; + } + + return ret; +} + +int ObTableApiModifyExecutor::generate_ins_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(*upd_ctdef.dins_ctdef_, + *upd_rtdef.dins_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_ins_ctdefs_, + upd_rtdef.related_ins_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + upd_rtdef.dins_rtdef_->related_ctdefs_ = &upd_ctdef.related_ins_ctdefs_; + upd_rtdef.dins_rtdef_->related_rtdefs_ = &upd_rtdef.related_ins_rtdefs_; + } + + return ret; +} + +int ObTableApiModifyExecutor::delete_upd_old_row_to_das(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value, + const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef, + sql::ObDMLRtCtx &dml_rtctx) +{ + int ret = OB_SUCCESS; + ObDASTabletLoc *tablet_loc = nullptr; + + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + if (OB_ISNULL(upd_ctdef.ddel_ctdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ddel_ctdef can't be null", K(ret)); + } else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) { + if (OB_FAIL(ObDASTaskFactory::alloc_das_rtdef(DAS_OP_TABLE_DELETE, + tb_ctx_.get_allocator(), + upd_rtdef.ddel_rtdef_))) { + LOG_WARN("fail to create das delete rtdef", K(ret)); + } else if (OB_FAIL(generate_del_rtdef_for_update(upd_ctdef, upd_rtdef))) { + LOG_WARN("fail to generate del rtdef for update", K(ret), K(upd_ctdef), K(upd_rtdef)); + } + } + ObChunkDatumStore::StoredRow* stored_row = nullptr; + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ddel_rtdef is null", K(ret)); + } else if (OB_FAIL(ObDMLService::delete_row(*upd_ctdef.ddel_ctdef_, + *upd_rtdef.ddel_rtdef_, + tablet_loc, + dml_rtctx, + upd_ctdef.old_row_, + stored_row))) { + LOG_WARN("fail to delete row with das", K(ret)); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::insert_upd_new_row_to_das(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef, + sql::ObDMLRtCtx &dml_rtctx) +{ + int ret = OB_SUCCESS; + ObDASTabletLoc *tablet_loc = nullptr; + + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + if (OB_ISNULL(upd_ctdef.dins_ctdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dins_ctdef_ can't be null", K(ret)); + } else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) { + if (OB_FAIL(ObDASTaskFactory::alloc_das_rtdef(DAS_OP_TABLE_INSERT, + tb_ctx_.get_allocator(), + upd_rtdef.dins_rtdef_))) { + LOG_WARN("fail to create das insert rtdef", K(ret)); + } else if (OB_FAIL(generate_ins_rtdef_for_update(upd_ctdef, upd_rtdef))) { + LOG_WARN("fail to generate del rtdef for update", K(ret), K(upd_ctdef), K(upd_rtdef)); + } + } + + clear_evaluated_flag(); + ObChunkDatumStore::StoredRow* stored_row = nullptr; + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dins_rtdef_ is null", K(ret)); + } else if (OB_FAIL(ObDMLService::insert_row(*upd_ctdef.dins_ctdef_, + *upd_rtdef.dins_rtdef_, + tablet_loc, + dml_rtctx, + upd_ctdef.new_row_, + stored_row))) { + LOG_WARN("fail to insert row with das", K(ret)); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::execute_das_task(ObDMLRtCtx &dml_rtctx, bool del_task_ahead) +{ + int ret = OB_SUCCESS; + + if (dml_rtctx.das_ref_.has_task()) { + if (del_task_ahead) { + if (OB_FAIL(dml_rtctx.das_ref_.pick_del_task_to_first())) { + LOG_WARN("fail to remove delete das task first", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) { + LOG_WARN("fail to execute all das task", K(ret)); + } + } + } + + return ret; +} + +void ObTableApiModifyExecutor::set_need_fetch_conflict(sql::ObDMLRtCtx &upd_rtctx, ObTableInsRtDef &ins_rtdef) +{ + ins_rtdef.das_rtdef_.need_fetch_conflict_ = true; + dml_rtctx_.set_non_sub_full_task(); + upd_rtctx.set_pick_del_task_first(); + upd_rtctx.set_non_sub_full_task(); +} + } // namespace table } // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_modify_executor.h b/src/observer/table/ob_table_modify_executor.h index ba27888758..1d4de7542f 100644 --- a/src/observer/table/ob_table_modify_executor.h +++ b/src/observer/table/ob_table_modify_executor.h @@ -96,9 +96,34 @@ protected: ObTableInsRtDef &ins_rtdef); int delete_row_to_das(const ObTableDelCtDef &del_ctdef, ObTableDelRtDef &del_rtdef); - // for replace & insert_up + // for replace & insert_up & ttl executor int get_next_conflict_rowkey(sql::DASTaskIter &task_iter, const sql::ObConflictChecker &conflict_checker); + // for htable + int modify_htable_timestamp(); + int fetch_conflict_rowkey(sql::ObConflictChecker &conflict_checker); + int reset_das_env(ObTableInsRtDef &ins_rtdef); + int check_whether_row_change(const ObChunkDatumStore::StoredRow &upd_old_row, + const ObChunkDatumStore::StoredRow &upd_new_row, + const ObTableUpdCtDef &upd_ctdef, + bool &is_row_changed); + int to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, + const ObRowkey &constraint_rowkey, + const ObTableUpdCtDef &upd_ctdef); + int generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef); + int generate_ins_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef); + int delete_upd_old_row_to_das(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value, + const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef, + sql::ObDMLRtCtx &dml_rtctx); + int insert_upd_new_row_to_das(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef, + sql::ObDMLRtCtx &dml_rtctx); + int execute_das_task(sql::ObDMLRtCtx &dml_rtctx, bool del_task_ahead); + void set_need_fetch_conflict(sql::ObDMLRtCtx &upd_rtctx,ObTableInsRtDef &ins_rtdef); protected: sql::ObDMLRtCtx dml_rtctx_; int64_t affected_rows_; diff --git a/src/observer/table/ob_table_op_wrapper.cpp b/src/observer/table/ob_table_op_wrapper.cpp index e07e89aabf..35cc3c55fc 100644 --- a/src/observer/table/ob_table_op_wrapper.cpp +++ b/src/observer/table/ob_table_op_wrapper.cpp @@ -18,6 +18,7 @@ #include "ob_htable_utils.h" #include "ob_htable_filter_operator.h" #include "ob_table_insert_up_executor.h" +#include "ttl/ob_table_ttl_executor.h" #include "ob_table_query_common.h" namespace oceanbase @@ -74,22 +75,40 @@ int ObTableOpWrapper::process_affected_entity(ObTableCtx &tb_ctx, { int ret = OB_SUCCESS; ObITableEntity *result_entity = nullptr; - if (TABLE_API_EXEC_INSERT_UP != spec.get_type()) { + if (TABLE_API_EXEC_INSERT_UP != spec.get_type() && TABLE_API_EXEC_TTL != spec.get_type()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid spec type", K(ret), K(spec.get_type())); } else if (OB_FAIL(op_result.get_entity(result_entity))) { LOG_WARN("fail to get result entity", K(ret), K(result_entity)); } else { ObIAllocator &allocator = tb_ctx.get_allocator(); - const ObTableApiInsertUpSpec &ins_up_spec = static_cast(spec); - const ObIArray &full_assign_exprs = ins_up_spec.get_ctdef().upd_ctdef_.full_assign_row_; - const ObIArray &ins_exprs = ins_up_spec.get_ctdef().ins_ctdef_.new_row_; + const ObIArray *full_assign_exprs = nullptr; + const ObIArray *ins_exprs = nullptr; + bool use_insert_expr = false; + if (TABLE_API_EXEC_INSERT_UP == spec.get_type()) { + const ObTableApiInsertUpSpec &ins_up_spec = static_cast(spec); + full_assign_exprs = &ins_up_spec.get_ctdef().upd_ctdef_.full_assign_row_; + ins_exprs = &ins_up_spec.get_ctdef().ins_ctdef_.new_row_; + use_insert_expr = !static_cast(executor).is_insert_duplicated(); + } else { + ObTableApiTTLExecutor &ttl_executor = static_cast(executor); + const ObTableApiTTLSpec &ttl_spec = static_cast(spec); + full_assign_exprs = &ttl_spec.get_ctdef().upd_ctdef_.full_assign_row_; + ins_exprs = &ttl_spec.get_ctdef().ins_ctdef_.new_row_; + use_insert_expr = !ttl_executor.is_insert_duplicated() || ttl_executor.is_expired(); + } const ObTableCtx::ObAssignIds &assign_ids = tb_ctx.get_assign_ids(); const int64_t N = assign_ids.count(); ObObj *obj_array = static_cast(allocator.alloc(sizeof(ObObj) * N)); if (OB_ISNULL(obj_array)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("faild to alloc memory for objs", K(ret)); + } else if (OB_ISNULL(full_assign_exprs)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("full assign exprs is null", K(ret)); + } else if (OB_ISNULL(ins_exprs)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert exprs is null", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { uint64_t idx = assign_ids.at(i).idx_; @@ -100,8 +119,7 @@ int ObTableOpWrapper::process_affected_entity(ObTableCtx &tb_ctx, LOG_WARN("column not exist", K(ret), K(column_id)); } else { ObObj &obj = obj_array[i]; - bool is_duplicated = static_cast(executor).is_insert_duplicated(); - ObExpr *rt_expr = is_duplicated ? full_assign_exprs.at(idx) : ins_exprs.at(idx); + ObExpr *rt_expr = use_insert_expr ? ins_exprs->at(idx) : full_assign_exprs->at(idx); ObDatum *datum = nullptr; const ObString &column_name = column_schema->get_column_name_str(); if (OB_FAIL(rt_expr->eval(executor.get_eval_ctx(), datum))) { @@ -324,6 +342,8 @@ int ObHTableDeleteExecutor::query_and_delete(const ObTableQuery &query) if (OB_FAIL(result_iter->get_next_result(one_result))) { if (OB_ITER_END != ret) { LOG_WARN("fail to get next result", K(ret)); + } else { + ret = OB_SUCCESS; } } else if (OB_ISNULL(one_result)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/observer/table/ob_table_query_and_mutate_processor.cpp b/src/observer/table/ob_table_query_and_mutate_processor.cpp index d8f98cad00..1dae911880 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.cpp +++ b/src/observer/table/ob_table_query_and_mutate_processor.cpp @@ -442,7 +442,10 @@ int ObTableQueryAndMutateP::refresh_query_range(const ObObj &new_q_obj) ctx_ranges.reset(); ObNewRange new_range = ranges.at(0); // shawdow copy tb_ctx_.set_limit(1); // 设置limit 1，只获取最新版本的数据 - if (FALSE_IT(new_range.start_key_.get_obj_ptr()[ObHTableConstants::COL_IDX_Q] = new_q_obj)) { + ObObj &q_obj = const_cast(new_q_obj); // 获取非const qualifier,为了varchar->varbinary + if (FALSE_IT(q_obj.set_varbinary(q_obj.get_varchar()))) { + // 将mutate的qualifier修改为varbinary + } else if (FALSE_IT(new_range.start_key_.get_obj_ptr()[ObHTableConstants::COL_IDX_Q] = new_q_obj)) { // 将mutate的qualifier作为新的扫描范围 } else if (FALSE_IT(new_range.end_key_.get_obj_ptr()[ObHTableConstants::COL_IDX_Q] = new_q_obj)) { // 将mutate的qualifier作为新的扫描范围 @@ -653,6 +656,7 @@ int ObTableQueryAndMutateP::execute_htable_mutation(ObTableQueryResultIterator * int ret = OB_SUCCESS; ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); const ObTableOperation &mutation = mutations.at(0); + uint64_t table_id = tb_ctx_.get_ref_table_id(); ObTableQueryResult *one_result = nullptr; // htable queryAndXXX only check one row @@ -662,36 +666,46 @@ int ObTableQueryAndMutateP::execute_htable_mutation(ObTableQueryResultIterator * } else { ret = OB_SUCCESS; one_result = &one_result_; // empty result is OK for APPEND and INCREMENT - switch(mutation.type()) { - case ObTableOperationType::DEL: { // checkAndDelete - stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_DELETE; - if (one_result->get_row_count() > 0) { // not empty result means check passed + bool expected_value_is_null = false; + bool result_value_is_null = false; + bool check_passed = false; + if (OB_FAIL(check_expected_value_is_null(result_iterator, expected_value_is_null))) { + LOG_WARN("fail to check whether expected value is null or not", K(ret)); + } else if ((one_result->get_row_count() > 0 && !expected_value_is_null) + || (one_result->get_row_count() == 0 && expected_value_is_null)) { + check_passed = true; + } else if (one_result->get_row_count() > 0 && expected_value_is_null) { + if (OB_FAIL(check_result_value_is_null(one_result, result_value_is_null))) { + LOG_WARN("fail to check whether result value is null or not", K(ret)); + } else if (result_value_is_null) { + check_passed = true; + } + } + if (OB_SUCC(ret) && check_passed) { + switch(mutation.type()) { + case ObTableOperationType::DEL: { // checkAndDelete + stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_DELETE; if (OB_FAIL(execute_htable_delete())) { LOG_WARN("fail to execute hatable delete", K(ret)); } else { affected_rows = 1; } + break; } - break; - } - case ObTableOperationType::INSERT_OR_UPDATE: { // checkAndPut - stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_PUT; - if (one_result->get_row_count() > 0) { // not empty result means check passed + case ObTableOperationType::INSERT_OR_UPDATE: { // checkAndPut + stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_PUT; if (OB_FAIL(execute_htable_put())) { LOG_WARN("fail to execute hatable put", K(ret)); } else { affected_rows = 1; } - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("insert or update with empty check result is not supported currently", K(ret)); + break; + } + default: { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported mutation type", K(ret), "type", mutation.type()); + break; } - break; - } - default: { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not supported mutation type", K(ret), "type", mutation.type()); - break; } } } @@ -707,24 +721,65 @@ int ObTableQueryAndMutateP::get_rowkey_column_names(ObIArray &names) if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", K(ret), K(tb_ctx_)); - } else { - const ObColumnSchemaV2 *column_schema = nullptr; - const ObRowkeyInfo &rowkey_info = table_schema->get_rowkey_info(); - const int64_t N = rowkey_info.get_size(); + } else if (OB_FAIL(ObTableQueryUtils::get_rowkey_column_names(*table_schema, names))) { + LOG_WARN("fail to get rowkey column names", K(ret)); + } - for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - uint64_t column_id = OB_INVALID_ID; - if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { - LOG_WARN("fail to get column id", K(ret), K(i), K(rowkey_info)); - } else if (NULL == (column_schema = table_schema->get_column_schema(column_id))){ - ret = OB_ERR_COLUMN_NOT_FOUND; - LOG_WARN("column not exists", K(ret), K(column_id)); - } else if (OB_FAIL(names.push_back(column_schema->get_column_name_str()))) { - LOG_WARN("fail to push back rowkey column name", K(ret), K(names)); + return ret; +} +int ObTableQueryAndMutateP::check_expected_value_is_null(ObTableQueryResultIterator *result_iter, bool &is_null) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator; + ObHTableFilterParser filter_parser; + hfilter::Filter *hfilter = nullptr; + is_null = false; + if (OB_ISNULL(result_iter)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("null result iterator", K(ret)); + } else { + ObHTableFilterOperator *filter_op = dynamic_cast(result_iter); + if (OB_ISNULL(filter_op)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("null filter operator", K(ret)); + } else { + hfilter = filter_op->get_hfiter(); + if OB_ISNULL(hfilter) { + // do nothing, filter string is empty, only htable key is specified + } else { + hfilter::CheckAndMutateFilter *query_and_mutate_filter = dynamic_cast(hfilter); + if (OB_ISNULL(query_and_mutate_filter)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("null query and mutate fiter", K(ret)); + } else { + is_null = query_and_mutate_filter->value_is_null(); + } } } } + return ret; +} +int ObTableQueryAndMutateP::check_result_value_is_null(ObTableQueryResult *query_result, bool &is_null_value) +{ + int ret = OB_SUCCESS; + const ObITableEntity *entity = nullptr; + is_null_value = false; + if (OB_ISNULL(query_result)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("null query result", K(ret)); + } else if (FALSE_IT(query_result->rewind())) { + } else if (OB_FAIL(query_result->get_next_entity(entity))) { + LOG_WARN("fail to get next entiry", K(ret)); + } else { + ObHTableCellEntity2 htable_cell(entity); + ObString value_str; + if (OB_FAIL(htable_cell.get_value(value_str))) { + LOG_WARN("fail to get value from htable cell", K(ret)); + } else if (value_str.empty()) { + is_null_value = true; + } + } return ret; } @@ -791,16 +846,26 @@ int ObTableQueryAndMutateP::execute_one_mutation(ObTableQueryResult &one_result, break; } case ObTableOperationType::INCREMENT: { - if (OB_FAIL(process_dml_op(*new_entity, tmp_affect_rows))) { - LOG_WARN("ail to execute table increment", K(ret)); + if (tb_ctx_.is_ttl_table()) { + ret = process_dml_op(*new_entity, tmp_affect_rows); + } else { + ret = process_dml_op(*new_entity, tmp_affect_rows); + } + if (OB_FAIL(ret)) { + LOG_WARN("fail to do increment", K(ret), K(tb_ctx_.is_ttl_table())); } else { affected_rows += tmp_affect_rows; } break; } case ObTableOperationType::APPEND: { - if (OB_FAIL(process_dml_op(*new_entity, tmp_affect_rows))) { - LOG_WARN("ail to execute table append", K(ret)); + if (tb_ctx_.is_ttl_table()) { + ret = process_dml_op(*new_entity, tmp_affect_rows); + } else { + ret = process_dml_op(*new_entity, tmp_affect_rows); + } + if (OB_FAIL(ret)) { + LOG_WARN("fail to do append", K(ret), K(tb_ctx_.is_ttl_table())); } else { affected_rows += tmp_affect_rows; } @@ -880,6 +945,7 @@ int ObTableQueryAndMutateP::try_process() // query_and_mutate request arg does not contain consisteny_level_ // @see ObTableQueryAndMutateRequest const ObTableConsistencyLevel consistency_level = ObTableConsistencyLevel::STRONG; + const ObTableQuery &query = arg_.query_and_mutate_.get_query(); ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); const ObTableOperation &mutation = mutations.at(0); observer::ObReqTimeGuard req_timeinfo_guard; // 引用cache资源必须加ObReqTimeGuard @@ -887,18 +953,26 @@ int ObTableQueryAndMutateP::try_process() ObTableApiSpec *scan_spec = nullptr; int64_t affected_rows = 0; const bool is_hkv = (ObTableEntityType::ET_HKV == arg_.entity_type_); + ObHTableLockHandle *lock_handle = nullptr; + uint64_t table_id = OB_INVALID_ID; if (OB_FAIL(init_scan_tb_ctx(cache_guard))) { LOG_WARN("fail to init scan table ctx", K(ret)); + } else if (FALSE_IT(table_id = tb_ctx_.get_ref_table_id())) { + } else if (is_hkv && OB_FAIL(HTABLE_LOCK_MGR->acquire_handle(lock_handle))) { + LOG_WARN("fail to get htable lock handle", K(ret)); + } else if (is_hkv && OB_FAIL(ObHTableUtils::lock_htable_row(table_id, query, *lock_handle, ObHTableLockMode::EXCLUSIVE))) { + LOG_WARN("fail to lock htable row", K(ret), K(table_id), K(query)); } else if (OB_FAIL(start_trans(false, /* is_readonly */ sql::stmt::T_UPDATE, consistency_level, - tb_ctx_.get_ref_table_id(), + table_id, tb_ctx_.get_ls_id(), get_timeout_ts()))) { LOG_WARN("fail to start readonly transaction", K(ret)); } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (is_hkv && FALSE_IT(lock_handle->set_tx_id(get_trans_desc()->tid()))) { } else if (OB_FAIL(cache_guard.get_spec(&tb_ctx_, scan_spec))) { LOG_WARN("fail to get scan spec from cache", K(ret)); } else if (is_hkv && ObTableOperationType::INCREMENT == mutation.type()) { @@ -957,7 +1031,7 @@ int ObTableQueryAndMutateP::try_process() bool need_rollback_trans = (OB_SUCCESS != ret); int tmp_ret = ret; const bool use_sync = true; - if (OB_FAIL(end_trans(need_rollback_trans, req_, get_timeout_ts(), use_sync))) { + if (OB_FAIL(end_trans(need_rollback_trans, req_, get_timeout_ts(), use_sync, lock_handle))) { LOG_WARN("failed to end trans", K(ret), "rollback", need_rollback_trans); } ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; diff --git a/src/observer/table/ob_table_query_and_mutate_processor.h b/src/observer/table/ob_table_query_and_mutate_processor.h index b2d34b8359..f1783780e2 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.h +++ b/src/observer/table/ob_table_query_and_mutate_processor.h @@ -77,6 +77,10 @@ private: const common::ObIArray &names, int64_t &affected_rows); int get_rowkey_column_names(common::ObIArray &names); + // check whether checkAndMutate's expected value is null or not + static int check_expected_value_is_null(table::ObTableQueryResultIterator *result_iter, bool &is_null); + // check value whether the value of first row of htable query result is null(empty string) or not + static int check_result_value_is_null(table::ObTableQueryResult *query_result, bool &is_null_value); private: template int process_dml_op(const table::ObITableEntity &new_entity, int64_t &affected_rows) diff --git a/src/observer/table/ob_table_query_common.cpp b/src/observer/table/ob_table_query_common.cpp index cabe8c6a58..b51e2e2bb7 100644 --- a/src/observer/table/ob_table_query_common.cpp +++ b/src/observer/table/ob_table_query_common.cpp @@ -66,7 +66,7 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator, int ret = OB_SUCCESS; ObTableQueryResultIterator *tmp_result_iter = nullptr; bool has_filter = (query.get_htable_filter().is_valid() || query.get_filter_string().length() > 0); - const ObString &schema_comment = tb_ctx.get_table_schema()->get_comment_str(); + const ObString &kv_attributes = tb_ctx.get_table_schema()->get_kv_attributes(); if (OB_FAIL(one_result.assign_property_names(tb_ctx.get_query_col_names()))) { LOG_WARN("fail to assign property names to one result", K(ret), K(tb_ctx)); @@ -86,10 +86,15 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator, } else { tmp_result_iter = htable_result_iter; ObHColumnDescriptor desc; - if (OB_FAIL(desc.from_string(schema_comment))) { - LOG_WARN("fail to parse hcolumn_desc from comment string", K(ret), K(schema_comment)); - } else if (desc.get_time_to_live() > 0) { - htable_result_iter->set_ttl(desc.get_time_to_live()); + if (OB_FAIL(desc.from_string(kv_attributes))) { + LOG_WARN("fail to parse hcolumn_desc from kv attributes", K(ret), K(kv_attributes)); + } else { + if (desc.get_time_to_live() > 0) { + htable_result_iter->set_ttl(desc.get_time_to_live()); + } + if (desc.get_max_version() > 0) { + htable_result_iter->set_max_version(desc.get_max_version()); + } } } } else { // tableapi @@ -134,5 +139,49 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator, return ret; } +int ObTableQueryUtils::get_rowkey_column_names(const ObTableSchema &table_schema, ObIArray &names) +{ + int ret = OB_SUCCESS; + const ObColumnSchemaV2 *column_schema = nullptr; + const ObRowkeyInfo &rowkey_info = table_schema.get_rowkey_info(); + const int64_t N = rowkey_info.get_size(); + + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + uint64_t column_id = OB_INVALID_ID; + if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { + LOG_WARN("fail to get column id", K(ret), K(i), K(rowkey_info)); + } else if (NULL == (column_schema = table_schema.get_column_schema(column_id))){ + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("column not exists", K(ret), K(column_id)); + } else if (OB_FAIL(names.push_back(column_schema->get_column_name_str()))) { + LOG_WARN("fail to push back rowkey column name", K(ret), K(names)); + } + } + + return ret; +} + +int ObTableQueryUtils::get_full_column_names(const ObTableSchema &table_schema, ObIArray &names) +{ + int ret = OB_SUCCESS; + const ObColumnSchemaV2 *column_schema = nullptr; + const ObRowkeyInfo &rowkey_info = table_schema.get_rowkey_info(); + const int64_t N = rowkey_info.get_size(); + + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + uint64_t column_id = OB_INVALID_ID; + if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { + LOG_WARN("fail to get column id", K(ret), K(i), K(rowkey_info)); + } else if (NULL == (column_schema = table_schema.get_column_schema(column_id))){ + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("column not exists", K(ret), K(column_id)); + } else if (OB_FAIL(names.push_back(column_schema->get_column_name_str()))) { + LOG_WARN("fail to push back rowkey column name", K(ret), K(names)); + } + } + + return ret; +} + } // end namespace table } // end namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_query_common.h b/src/observer/table/ob_table_query_common.h index be4f642bbb..0983b73c78 100644 --- a/src/observer/table/ob_table_query_common.h +++ b/src/observer/table/ob_table_query_common.h @@ -30,6 +30,9 @@ public: ObTableQueryResult &one_result, const ObTableCtx &tb_ctx, ObTableQueryResultIterator *&result_iter); + static int get_rowkey_column_names(const ObTableSchema &table_schema, ObIArray &names); + static int get_full_column_names(const ObTableSchema &table_schema, ObIArray &names); + private: static int check_htable_query_args(const ObTableQuery &query, const ObTableCtx &tb_ctx); private: diff --git a/src/observer/table/ob_table_replace_executor.cpp b/src/observer/table/ob_table_replace_executor.cpp index 68d7ee6f7e..3d230952ba 100644 --- a/src/observer/table/ob_table_replace_executor.cpp +++ b/src/observer/table/ob_table_replace_executor.cpp @@ -107,13 +107,9 @@ int ObTableApiReplaceExecutor::get_next_row_from_child() int ObTableApiReplaceExecutor::load_replace_rows(bool &is_iter_end) { int ret = OB_SUCCESS; - int64_t row_cnt = 0; const ObTableReplaceCtDef &ctdef = replace_spec_.get_ctdef(); - int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_REPLACE_BATCH_ROW_COUNT); - int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? - simulate_batch_row_cnt : DEFAULT_REPLACE_BATCH_ROW_COUNT; - while (OB_SUCC(ret) && ++row_cnt < default_row_batch_cnt) { + while (OB_SUCC(ret)) { if (OB_FAIL(get_next_row_from_child())) { if (OB_ITER_END != ret) { LOG_WARN("fail to load next row from child", K(ret)); @@ -147,45 +143,6 @@ int ObTableApiReplaceExecutor::post_das_task() return ret; } -int ObTableApiReplaceExecutor::fetch_conflict_rowkey() -{ - int ret = OB_SUCCESS; - bool got_row = false; - DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); - - while (OB_SUCC(ret) && !task_iter.is_end()) { - if (OB_FAIL(get_next_conflict_rowkey(task_iter, conflict_checker_))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next conflict rowkey from das_result", K(ret)); - } - } else if (OB_FAIL(conflict_checker_.build_primary_table_lookup_das_task())) { - LOG_WARN("fail to build lookup_das_task", K(ret)); - } - } - - ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); - return ret; -} - -int ObTableApiReplaceExecutor::reset_das_env() -{ - int ret = OB_SUCCESS; - - // 释放第一次try insert的das task - if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { - LOG_WARN("close all das task failed", K(ret)); - } else { - dml_rtctx_.das_ref_.reuse(); - } - - // 因为第二次插入不需要fetch conflict result了，如果有conflict - // 就说明replace into的某些逻辑处理有问题 - replace_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = false; - replace_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_ = false; - - return ret; -} - int ObTableApiReplaceExecutor::check_values(bool &is_equal, const ObChunkDatumStore::StoredRow *replace_row, const ObChunkDatumStore::StoredRow *delete_row) @@ -321,9 +278,9 @@ int ObTableApiReplaceExecutor::get_next_row() LOG_WARN("fail to post all das task", K(ret)); } else if (!is_duplicated()) { LOG_DEBUG("try insert is not duplicated", K(ret)); - } else if (OB_FAIL(fetch_conflict_rowkey())) { + } else if (OB_FAIL(fetch_conflict_rowkey(conflict_checker_))) { LOG_WARN("fail to fetch conflict row", K(ret)); - } else if (OB_FAIL(reset_das_env())) { + } else if (OB_FAIL(reset_das_env(replace_rtdef_.ins_rtdef_))) { // 这里需要reuse das 相关信息 LOG_WARN("fail to reset das env", K(ret)); } else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(exec_ctx_, savepoint_no))) { diff --git a/src/observer/table/ob_table_replace_executor.h b/src/observer/table/ob_table_replace_executor.h index 2ba9315713..d3244ac47b 100644 --- a/src/observer/table/ob_table_replace_executor.h +++ b/src/observer/table/ob_table_replace_executor.h @@ -89,8 +89,6 @@ private: int load_replace_rows(bool &is_iter_end); int get_next_row_from_child(); int post_das_task(); - int fetch_conflict_rowkey(); - int reset_das_env(); int check_values(bool &is_equal, const ObChunkDatumStore::StoredRow *replace_row, const ObChunkDatumStore::StoredRow *delete_row); diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index 7cff3cffec..dc8e4fe5c4 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -363,7 +363,7 @@ int ObTableApiProcessorBase::init_read_trans(const ObTableConsistencyLevel consi if (OB_FAIL(txs->acquire_tx(trans_desc_, session().get_sessid()))) { LOG_WARN("failed to acquire tx desc", K(ret)); - } else if (OB_FAIL(setup_tx_snapshot_(*trans_desc_, strong_read, ls_id, timeout_ts))) { + } else if (OB_FAIL(setup_tx_snapshot_(*trans_desc_, tx_snapshot_, strong_read, ls_id, timeout_ts))) { LOG_WARN("setup txn snapshot fail", K(ret), KPC_(trans_desc), K(strong_read), K(ls_id), K(timeout_ts)); txs->release_tx(*trans_desc_); trans_desc_ = NULL; @@ -382,6 +382,7 @@ void ObTableApiProcessorBase::release_read_trans() } int ObTableApiProcessorBase::setup_tx_snapshot_(transaction::ObTxDesc &trans_desc, + transaction::ObTxReadSnapshot &tx_snapshot, const bool strong_read, const share::ObLSID &ls_id, const int64_t timeout_ts) @@ -390,10 +391,10 @@ int ObTableApiProcessorBase::setup_tx_snapshot_(transaction::ObTxDesc &trans_des transaction::ObTransService *txs = MTL(transaction::ObTransService*); if (strong_read) { if (ls_id.is_valid()) { - if (OB_FAIL(txs->get_ls_read_snapshot(trans_desc, transaction::ObTxIsolationLevel::RC, ls_id, timeout_ts, tx_snapshot_))) { + if (OB_FAIL(txs->get_ls_read_snapshot(trans_desc, transaction::ObTxIsolationLevel::RC, ls_id, timeout_ts, tx_snapshot))) { LOG_WARN("fail to get LS read snapshot", K(ret)); } - } else if (OB_FAIL(txs->get_read_snapshot(trans_desc, transaction::ObTxIsolationLevel::RC, timeout_ts, tx_snapshot_))) { + } else if (OB_FAIL(txs->get_read_snapshot(trans_desc, transaction::ObTxIsolationLevel::RC, timeout_ts, tx_snapshot))) { LOG_WARN("fail to get global read snapshot", K(ret)); } } else { @@ -404,21 +405,31 @@ int ObTableApiProcessorBase::setup_tx_snapshot_(transaction::ObTxDesc &trans_des weak_read_snapshot))) { LOG_WARN("fail to get weak read snapshot", K(ret)); } else { - tx_snapshot_.init_weak_read(weak_read_snapshot); + tx_snapshot.init_weak_read(weak_read_snapshot); } } return ret; } int ObTableApiProcessorBase::start_trans(bool is_readonly, const sql::stmt::StmtType stmt_type, - const ObTableConsistencyLevel consistency_level, uint64_t table_id, - const ObLSID &ls_id, int64_t timeout_ts) + const ObTableConsistencyLevel consistency_level, + uint64_t table_id, const ObLSID &ls_id, int64_t timeout_ts) +{ + UNUSED(stmt_type); + return start_trans_(is_readonly, trans_desc_, tx_snapshot_, consistency_level, + trans_state_ptr_, table_id, ls_id, timeout_ts); +} + +int ObTableApiProcessorBase::start_trans_(bool is_readonly, + transaction::ObTxDesc *&trans_desc, + transaction::ObTxReadSnapshot &tx_snapshot, + const ObTableConsistencyLevel consistency_level, + sql::TransState *trans_state_ptr, + uint64_t table_id, const ObLSID &ls_id, int64_t timeout_ts) { - UNUSEDx(stmt_type); int ret = OB_SUCCESS; NG_TRACE(T_start_trans_begin); - const uint64_t tenant_id = credential_.tenant_id_; bool strong_read = ObTableConsistencyLevel::STRONG == consistency_level; if ((!is_readonly) && (ObTableConsistencyLevel::EVENTUAL == consistency_level)) { ret = OB_NOT_SUPPORTED; @@ -436,32 +447,34 @@ int ObTableApiProcessorBase::start_trans(bool is_readonly, const sql::stmt::Stmt tx_param.isolation_ = transaction::ObTxIsolationLevel::RC; tx_param.cluster_id_ = ObServerConfig::get_instance().cluster_id; tx_param.timeout_us_ = std::max(0l, timeout_ts - ObClockGenerator::getClock()); - if (true == trans_state_ptr_->is_start_trans_executed()) { + if (true == trans_state_ptr->is_start_trans_executed()) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("start_trans is executed", K(ret)); } else { - if (OB_FAIL(txs->acquire_tx(trans_desc_, session().get_sessid()))) { + if (OB_FAIL(txs->acquire_tx(trans_desc, session().get_sessid()))) { LOG_WARN("failed to acquire tx desc", K(ret)); - } else if (OB_FAIL(txs->start_tx(*trans_desc_, tx_param))) { - LOG_WARN("failed to start trans", K(ret), KPC_(trans_desc)); - txs->release_tx(*trans_desc_); - trans_desc_ = NULL; + } else if (OB_FAIL(txs->start_tx(*trans_desc, tx_param))) { + LOG_WARN("failed to start trans", K(ret), KPC(trans_desc)); + txs->release_tx(*trans_desc); + trans_desc = NULL; } - trans_state_ptr_->set_start_trans_executed(OB_SUCC(ret)); + trans_state_ptr->set_start_trans_executed(OB_SUCC(ret)); } } NG_TRACE(T_start_trans_end); // 2. acquire snapshot if (OB_SUCC(ret)) { - if (OB_FAIL(setup_tx_snapshot_(*trans_desc_, strong_read, ls_id, timeout_ts))) { - LOG_WARN("setup txn snapshot fail", K(ret), KPC_(trans_desc), K(strong_read), K(ls_id), K(timeout_ts)); + if (OB_FAIL(setup_tx_snapshot_(*trans_desc, tx_snapshot, strong_read, ls_id, timeout_ts))) { + LOG_WARN("setup txn snapshot fail", K(ret), KPC(trans_desc), K(strong_read), K(ls_id), K(timeout_ts)); } } return ret; } +// NOTE: the lock handle pointer can be converted to bool parameter use_sync, +// be careful whenever you want to pass the lock handle int ObTableApiProcessorBase::end_trans(bool is_rollback, rpc::ObRequest *req, int64_t timeout_ts, - bool use_sync /*=false*/) + bool use_sync /*=false*/, ObHTableLockHandle *lock_handle /*nullptr*/) { int ret = OB_SUCCESS; int end_ret = OB_SUCCESS; @@ -478,12 +491,12 @@ int ObTableApiProcessorBase::end_trans(bool is_rollback, rpc::ObRequest *req, in NG_TRACE(T_end_trans_begin); if (trans_state_ptr_->is_start_trans_executed() && trans_state_ptr_->is_start_trans_success()) { if (trans_desc_->is_rdonly() || use_sync) { - ret = sync_end_trans(is_rollback, timeout_ts); + ret = sync_end_trans(is_rollback, timeout_ts, lock_handle); } else { if (is_rollback) { - ret = sync_end_trans(true, timeout_ts); + ret = sync_end_trans(true, timeout_ts, lock_handle); } else { - ret = async_commit_trans(req, timeout_ts); + ret = async_commit_trans(req, timeout_ts, lock_handle); } } trans_state_ptr_->clear_start_trans_executed(); @@ -493,36 +506,43 @@ int ObTableApiProcessorBase::end_trans(bool is_rollback, rpc::ObRequest *req, in return ret; } -int ObTableApiProcessorBase::sync_end_trans(bool is_rollback, int64_t timeout_ts) +int ObTableApiProcessorBase::sync_end_trans(bool is_rollback, int64_t timeout_ts, ObHTableLockHandle *lock_handle /*nullptr*/) +{ + return sync_end_trans_(is_rollback, trans_desc_, timeout_ts, lock_handle); +} + +int ObTableApiProcessorBase::sync_end_trans_(bool is_rollback, transaction::ObTxDesc *&trans_desc, + int64_t timeout_ts, ObHTableLockHandle *lock_handle /*nullptr*/) { int ret = OB_SUCCESS; transaction::ObTransService *txs = MTL(transaction::ObTransService*); const int64_t stmt_timeout_ts = timeout_ts; if (is_rollback) { - if (OB_FAIL(txs->rollback_tx(*trans_desc_))) { - LOG_WARN("fail rollback trans when session terminate", K(ret), KPC_(trans_desc)); + if (OB_FAIL(txs->rollback_tx(*trans_desc))) { + LOG_WARN("fail rollback trans when session terminate", K(ret), KPC(trans_desc)); } - } else { + } else if (OB_FAIL(txs->commit_tx(*trans_desc, stmt_timeout_ts))) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_committing); - if (OB_FAIL(txs->commit_tx(*trans_desc_, stmt_timeout_ts))) { - LOG_WARN("fail commit trans when session terminate", - K(ret), KPC_(trans_desc), K(stmt_timeout_ts)); - } + LOG_WARN("fail commit trans when session terminate", + K(ret), KPC(trans_desc), K(stmt_timeout_ts)); } int tmp_ret = ret; - if (OB_FAIL(txs->release_tx(*trans_desc_))) { - LOG_ERROR("release tx failed", K(ret), KPC_(trans_desc)); + if (OB_FAIL(txs->release_tx(*trans_desc))) { + LOG_ERROR("release tx failed", K(ret), KPC(trans_desc)); + } + if (lock_handle != nullptr) { + HTABLE_LOCK_MGR->release_handle(*lock_handle); } ret = tmp_ret == OB_SUCCESS ? ret : tmp_ret; - trans_desc_ = NULL; + trans_desc = NULL; LOG_DEBUG("ObTableApiProcessorBase::sync_end_trans", K(ret), K(is_rollback), K(stmt_timeout_ts)); return ret; } -int ObTableApiProcessorBase::async_commit_trans(rpc::ObRequest *req, int64_t timeout_ts) +int ObTableApiProcessorBase::async_commit_trans(rpc::ObRequest *req, int64_t timeout_ts, ObHTableLockHandle *lock_handle /*nullptr*/) { int ret = OB_SUCCESS; transaction::ObTransService *txs = MTL(transaction::ObTransService*); @@ -538,6 +558,7 @@ int ObTableApiProcessorBase::async_commit_trans(rpc::ObRequest *req, int64_t tim } callback.set_is_need_rollback(is_rollback); callback.set_end_trans_type(sql::ObExclusiveEndTransCallback::END_TRANS_TYPE_IMPLICIT); + callback.set_lock_handle(lock_handle); callback.handout(); callback.set_tx_desc(trans_desc_); const int64_t stmt_timeout_ts = timeout_ts; diff --git a/src/observer/table/ob_table_rpc_processor.h b/src/observer/table/ob_table_rpc_processor.h index ae4234b6e8..8c586a6acd 100644 --- a/src/observer/table/ob_table_rpc_processor.h +++ b/src/observer/table/ob_table_rpc_processor.h @@ -21,6 +21,7 @@ #include "ob_table_service.h" #include "sql/monitor/ob_exec_stat.h" #include "share/table/ob_table.h" +#include "ob_htable_lock_mgr.h" namespace oceanbase { namespace table @@ -115,7 +116,19 @@ public: int start_trans(bool is_readonly, const sql::stmt::StmtType stmt_type, const table::ObTableConsistencyLevel consistency_level, uint64_t table_id, const share::ObLSID &ls_id, int64_t timeout_ts); - int end_trans(bool is_rollback, rpc::ObRequest *req, int64_t timeout_ts, bool use_sync = false); + int end_trans(bool is_rollback, rpc::ObRequest *req, int64_t timeout_ts, + bool use_sync = false, table::ObHTableLockHandle *lock_handle = nullptr); + static int start_trans_(bool is_readonly, transaction::ObTxDesc*& trans_desc, + transaction::ObTxReadSnapshot &tx_snapshot, + const ObTableConsistencyLevel consistency_level, + sql::TransState *trans_state_ptr, + uint64_t table_id, const share::ObLSID &ls_id, int64_t timeout_ts); + int sync_end_trans(bool is_rollback, int64_t timeout_ts, table::ObHTableLockHandle *lock_handle = nullptr); + static int sync_end_trans_(bool is_rollback, + transaction::ObTxDesc *&trans_desc, + int64_t timeout_ts, + table::ObHTableLockHandle *lock_handle = nullptr); + // for get int init_read_trans(const table::ObTableConsistencyLevel consistency_level, const share::ObLSID &ls_id, @@ -146,9 +159,12 @@ protected: virtual int check_table_index_supported(uint64_t table_id, bool &is_supported); private: - int setup_tx_snapshot_(transaction::ObTxDesc &trans_desc, const bool strong_read, const share::ObLSID &ls_id, const int64_t timeout_ts); - int async_commit_trans(rpc::ObRequest *req, int64_t timeout_ts); - int sync_end_trans(bool is_rollback, int64_t timeout_ts); + int async_commit_trans(rpc::ObRequest *req, int64_t timeout_ts, table::ObHTableLockHandle *lock_handle = nullptr); + static int setup_tx_snapshot_(transaction::ObTxDesc &trans_desc, + transaction::ObTxReadSnapshot &tx_snapshot, + const bool strong_read, + const share::ObLSID &ls_id, + const int64_t timeout_ts); protected: const ObGlobalContext &gctx_; ObTableService *table_service_; diff --git a/src/observer/table/ob_table_scan_executor.cpp b/src/observer/table/ob_table_scan_executor.cpp index 6bfa0a4c41..2ce3e5318e 100644 --- a/src/observer/table/ob_table_scan_executor.cpp +++ b/src/observer/table/ob_table_scan_executor.cpp @@ -190,6 +190,7 @@ int ObTableApiScanExecutor::get_next_row_with_das() int ret = OB_SUCCESS; bool got_row = false; while (OB_SUCC(ret) && !got_row) { + bool filter = false; clear_evaluated_flag(); if (OB_FAIL(scan_result_.get_next_row())) { if (OB_ITER_END == ret) { @@ -201,6 +202,10 @@ int ObTableApiScanExecutor::get_next_row_with_das() } else { LOG_WARN("get next row from das result failed", K(ret)); } + } else if (OB_FAIL(check_filter(filter))) { + LOG_WARN("fail to check row filtered", K(ret)); + } else if (filter) { + LOG_DEBUG("the row is filtered", K(ret)); } else { ++input_row_cnt_; ++output_row_cnt_; @@ -210,6 +215,26 @@ int ObTableApiScanExecutor::get_next_row_with_das() return ret; } +int ObTableApiScanExecutor::check_filter(bool &filter) +{ + int ret = OB_SUCCESS; + const ObIArray &exprs = scan_spec_.get_ctdef().filter_exprs_; + ObDatum *datum = NULL; + filter = false; + + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count() && !filter; i++) { + if (OB_FAIL(exprs.at(i)->eval(eval_ctx_, datum))) { + LOG_WARN("fail to eval filter expr", K(ret), K(*exprs.at(i))); + } else if (tb_ctx_.is_ttl_table()) { + filter = (!datum->is_null() && datum->get_bool()); // ttl场景下，过期表达式不过滤is_null + } else { + filter = datum->get_bool(); + } + } + + return ret; +} + int ObTableApiScanExecutor::open() { int ret = OB_SUCCESS; @@ -346,5 +371,14 @@ int ObTableApiScanRowIterator::close() return ret; } +void ObTableApiScanExecutor::clear_evaluated_flag() +{ + const ExprFixedArray &filter_exprs = get_spec().get_ctdef().filter_exprs_; + for (int64_t i = 0; i < filter_exprs.count(); i++) { + ObSQLUtils::clear_expr_eval_flags(*filter_exprs.at(i), eval_ctx_); + } + ObTableApiExecutor::clear_evaluated_flag(); +} + } // namespace table } // namespace oceanbase diff --git a/src/observer/table/ob_table_scan_executor.h b/src/observer/table/ob_table_scan_executor.h index 93a2077965..472d0bbbba 100644 --- a/src/observer/table/ob_table_scan_executor.h +++ b/src/observer/table/ob_table_scan_executor.h @@ -52,6 +52,7 @@ public: int get_next_row() override; int close() override; void destroy() override {} + virtual void clear_evaluated_flag() override; public: OB_INLINE const ObTableApiScanSpec& get_spec() const { return scan_spec_; } OB_INLINE void reset() @@ -78,6 +79,7 @@ private: int prepare_das_task(); int do_table_scan(); int get_next_row_with_das(); + int check_filter(bool &filter); private: DISALLOW_COPY_AND_ASSIGN(ObTableApiScanExecutor); }; @@ -91,9 +93,9 @@ public: } virtual ~ObTableApiScanRowIterator() {}; public: - int open(ObTableApiScanExecutor *executor); - int get_next_row(common::ObNewRow *&row); - int close(); + virtual int open(ObTableApiScanExecutor *executor); + virtual int get_next_row(common::ObNewRow *&row); + virtual int close(); private: ObTableApiScanExecutor *scan_executor_; private: diff --git a/src/observer/table/ob_table_session_pool.cpp b/src/observer/table/ob_table_session_pool.cpp index 00eb8bff6c..fd1ed0b6ec 100644 --- a/src/observer/table/ob_table_session_pool.cpp +++ b/src/observer/table/ob_table_session_pool.cpp @@ -138,6 +138,25 @@ int ObTableApiSessPoolMgr::update_sess(ObTableApiCredential &credential) return ret; } +int ObTableApiSessPoolMgr::create_pool_if_not_exists(int64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObTableApiSessPoolGuard guard; + if (OB_FAIL(get_session_pool(tenant_id, guard))) { + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(extend_sess_pool(tenant_id, guard))) { + LOG_WARN("fail to extend sess pool", K(ret), K(tenant_id)); + } else { + LOG_INFO("success to extend sess pool", K(ret), K(tenant_id)); + } + } else { + LOG_WARN("fait to get session pool", K(ret), K(tenant_id)); + } + } + + return ret; +} + int ObTableApiSessPoolMgr::extend_sess_pool(uint64_t tenant_id, ObTableApiSessPoolGuard &guard) { @@ -555,7 +574,7 @@ int ObTableApiSessPool::create_and_add_node(ObTableApiCredential &credential) // 1. login时调用 // 2. 不存在，创建; 存在，旧的移动到淘汰链表, 添加新的node -int ObTableApiSessPool::update_sess(ObTableApiCredential &credential) +int ObTableApiSessPool::update_sess(ObTableApiCredential &credential, bool replace_old_node) { int ret = OB_SUCCESS; @@ -569,7 +588,7 @@ int ObTableApiSessPool::update_sess(ObTableApiCredential &credential) } else { LOG_WARN("fail to get session node", K(ret), K(key)); } - } else { // exist, 替换node，old node移动到淘汰链表等待淘汰 + } else if (replace_old_node) { // exist, 替换node，old node移动到淘汰链表等待淘汰 if (OB_FAIL(replace_sess(credential))) { LOG_WARN("fail to replace session node", K(ret), K(credential)); } diff --git a/src/observer/table/ob_table_session_pool.h b/src/observer/table/ob_table_session_pool.h index 35ace17916..72d9ad18d1 100644 --- a/src/observer/table/ob_table_session_pool.h +++ b/src/observer/table/ob_table_session_pool.h @@ -67,6 +67,7 @@ public: int get_session_pool(uint64_t tenant_id, ObTableApiSessPoolGuard &guard); int get_sess_info(ObTableApiCredential &credential, ObTableApiSessGuard &guard); int update_sess(ObTableApiCredential &credential); + int create_pool_if_not_exists(int64_t tenant_id); private: int extend_sess_pool(uint64_t tenant_id, ObTableApiSessPoolGuard &guard); int get_or_create_sess_pool(ObTableApiCredential &credential, ObTableApiSessPoolGuard &guard); @@ -125,7 +126,7 @@ public: bool is_deleted() { return ATOMIC_LOAD(&is_deleted_); } bool is_empty() const { return key_node_map_.empty(); } int get_sess_info(ObTableApiCredential &credential, ObTableApiSessGuard &guard); - int update_sess(ObTableApiCredential &credential); + int update_sess(ObTableApiCredential &credential, bool replace_old_node = true); // 将过期的node移动到retired_nodes_ int move_sess_to_retired_list(); int evict_retired_sess(); diff --git a/src/observer/table/ttl/ob_table_ttl_executor.cpp b/src/observer/table/ttl/ob_table_ttl_executor.cpp new file mode 100644 index 0000000000..5aeb9f2766 --- /dev/null +++ b/src/observer/table/ttl/ob_table_ttl_executor.cpp @@ -0,0 +1,447 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_ttl_executor.h" +#include "src/observer/table/ob_table_cg_service.h" +#include "lib/utility/ob_tracepoint.h" +#include "sql/das/ob_das_insert_op.h" +#include "sql/engine/dml/ob_dml_service.h" +#include "src/observer/table/ob_htable_utils.h" +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ + +int ObTableApiTTLExecutor::generate_ttl_rtdef(const ObTableTTLCtDef &ctdef, ObTableTTLRtDef &rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_ins_rtdef(ctdef.ins_ctdef_, + rtdef.ins_rtdef_))) { + LOG_WARN("fail to generate insert rtdef", K(ret)); + } else if (OB_FAIL(generate_del_rtdef(ctdef.del_ctdef_, + rtdef.del_rtdef_))) { + LOG_WARN("fail to generate delete rtdef", K(ret)); + } else if (OB_FAIL(generate_upd_rtdef(ctdef.upd_ctdef_, + rtdef.upd_rtdef_))) { + LOG_WARN("fail to generate update rtdef", K(ret)); + } + + return ret; +} + +int ObTableApiTTLExecutor::open() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_ttl_rtdef(ttl_spec_.get_ctdef(), ttl_rtdef_))) { + LOG_WARN("fail to generate ttl rtdef", K(ret)); + } else { + ObDASTabletLoc *tablet_loc = nullptr; + ObDASTableLoc *table_loc = ttl_rtdef_.ins_rtdef_.das_rtdef_.table_loc_; + if (OB_ISNULL(table_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table location is invalid", K(ret)); + } else if (OB_FAIL(conflict_checker_.init_conflict_checker(ttl_spec_.get_expr_frame_info(), table_loc))) { + LOG_WARN("fail to init conflict_checker", K(ret)); + } else if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + conflict_checker_.set_local_tablet_loc(tablet_loc); + // init update das_ref + ObMemAttr mem_attr; + mem_attr.tenant_id_ = tb_ctx_.get_session_info().get_effective_tenant_id(); + mem_attr.label_ = "TableApiTTL"; + upd_rtctx_.das_ref_.set_expr_frame_info(ttl_spec_.get_expr_frame_info()); + upd_rtctx_.das_ref_.set_mem_attr(mem_attr); + upd_rtctx_.das_ref_.set_execute_directly(true); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (tb_ctx_.is_htable() && OB_FAIL(modify_htable_timestamp())) { + LOG_WARN("fail to modify htable timestamp", K(ret)); + } + + return ret; +} + +int ObTableApiTTLExecutor::refresh_exprs_frame(const ObTableEntity *entity) +{ + int ret = OB_SUCCESS; + const ObTableTTLCtDef &ctdef = ttl_spec_.get_ctdef(); + const ObTableInsCtDef &ins_ctdef = ctdef.ins_ctdef_; + const ObTableUpdCtDef &upd_ctdef = ctdef.upd_ctdef_; + + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::refresh_ttl_exprs_frame(tb_ctx_, + ins_ctdef.new_row_, + upd_ctdef.delta_exprs_, + *entity))) { + LOG_WARN("fail to refresh ttl exprs frame", K(ret), K(*entity)); + } + + return ret; +} + +int ObTableApiTTLExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + if (cur_idx_ >= 1) { + ret = OB_ITER_END; + } else if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } + + return ret; +} + +int ObTableApiTTLExecutor::try_insert_row() +{ + int ret = OB_SUCCESS; + const ObTableTTLCtDef &ctdef = ttl_spec_.get_ctdef(); + + while (OB_SUCC(ret)) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row from child", K(ret)); + } + } else if (OB_FAIL(insert_row_to_das(ctdef.ins_ctdef_, ttl_rtdef_.ins_rtdef_))) { + LOG_WARN("fail to insert row to das", K(ret)); + } else { + cur_idx_++; + } + } + + if (OB_ITER_END == ret && OB_FAIL(execute_das_task(dml_rtctx_, false/* del_task_ahead */))) { + LOG_WARN("fail to execute das task"); + } + + return ret; +} + +int ObTableApiTTLExecutor::check_expired(bool &is_expired) +{ + int ret = OB_SUCCESS; + ObExpr *expr = ttl_spec_.get_ctdef().expire_expr_; + ObDatum *datum = nullptr; + + if (OB_ISNULL(expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("expire expr is null", K(ret)); + } else if (OB_FAIL(expr->eval(eval_ctx_, datum))) { + LOG_WARN("fail to eval expire expr", K(ret), K(*expr)); + } else { + is_expired = (!datum->is_null() && datum->get_bool()); + } + + return ret; +} + +int ObTableApiTTLExecutor::do_delete() +{ + int ret = OB_SUCCESS; + const ObTableTTLCtDef &ctdef = ttl_spec_.get_ctdef(); + + if (OB_FAIL(delete_row_to_das(ctdef.del_ctdef_, ttl_rtdef_.del_rtdef_))) { + LOG_WARN("fail to delete row to das", K(ret), K(ctdef), K_(ttl_rtdef)); + } + + return ret; +} + +int ObTableApiTTLExecutor::do_insert() +{ + int ret = OB_SUCCESS; + const ObTableTTLCtDef &ctdef = ttl_spec_.get_ctdef(); + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + // do_insert前被conflict checker刷成了旧行，需要重新刷一遍 + if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } else if (OB_FAIL(insert_row_to_das(ctdef.ins_ctdef_, ttl_rtdef_.ins_rtdef_))) { + LOG_WARN("fail to insert row to das", K(ret), K(ctdef), K_(ttl_rtdef)); + } + + return ret; +} + +int ObTableApiTTLExecutor::update_row_to_conflict_checker() +{ + int ret = OB_SUCCESS; + ObSEArray constraint_values; + ObChunkDatumStore::StoredRow *insert_row = nullptr; + ObTableUpdRtDef &upd_rtdef = ttl_rtdef_.upd_rtdef_; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + const ObExprPtrIArray &new_row_exprs = get_primary_table_insert_row(); + + if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } else if (OB_FAIL(ObChunkDatumStore::StoredRow::build(insert_row, new_row_exprs, eval_ctx_, allocator_))) { + LOG_WARN("fail to build stored row", K(ret), K(new_row_exprs)); + } else if (OB_ISNULL(insert_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert row is NULL", K(ret)); + } else if (OB_FAIL(conflict_checker_.check_duplicate_rowkey(insert_row, constraint_values, true))) { + LOG_WARN("fail to check duplicated key", K(ret), KPC(insert_row)); + } else { + upd_rtdef.found_rows_++; + const ObChunkDatumStore::StoredRow *upd_new_row = insert_row; + const ObChunkDatumStore::StoredRow *upd_old_row = constraint_values.at(0).current_datum_row_; // 这里只取第一行是和mysql对齐的 + if (OB_ISNULL(upd_old_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("upd_old_row is NULL", K(ret)); + } else if (OB_FAIL(check_whether_row_change(*upd_old_row, + *upd_new_row, + ttl_spec_.get_ctdef().upd_ctdef_, + is_row_changed_))) { + LOG_WARN("fail to check whether row change", K(ret)); + } else if (is_row_changed_) { + // do update + clear_evaluated_flag(); + if (OB_FAIL(conflict_checker_.update_row(upd_new_row, upd_old_row))) { + LOG_WARN("fail to update row in conflict_checker", K(ret), KPC(upd_new_row), KPC(upd_old_row)); + } + } + } + + return ret; +} + +// create table t(c1 int primary key, c2 int, c3 int, +// unique key idx0(c2), unique key idx1(c3)); +// insert into t values(1, 1, 1),(2, 2, 2),(3, 3, 3); +// insert into t values(3,1,2) ON DUPLICATE KEY UPDATE c1=4; +// 执行insert后map中有3个元素： +// 1->(base:1, 1, 1 curr:1, 1, 1), +// 2->(base:2, 2, 2 curr:2, 2, 2), +// 3->(base:3, 3, 3 curr:3, 3, 3); +// 执行conflict_checker_.update_row后: +// 1->(base:1, 1, 1 curr:nul), +// 2->(base:2, 2, 2 curr:nul), +// 3->(base:3, 3, 3 curr:4, 3, 3); +// delete->base:3, 3, 3, insert->curr:4, 3, 3 +// result: +// +----+------+------+ +// | c1 | c2 | c3 | +// +----+------+------+ +// | 1 | 1 | 1 | +// | 2 | 2 | 2 | +// | 4 | 3 | 3 | +// +----+------+------+ +int ObTableApiTTLExecutor::update_row_to_das() +{ + int ret = OB_SUCCESS; + ObConflictRowMap *primary_map = nullptr; + OZ(conflict_checker_.get_primary_table_map(primary_map)); + CK(OB_NOT_NULL(primary_map)); + ObConflictRowMap::iterator start = primary_map->begin(); + ObConflictRowMap::iterator end = primary_map->end(); + for (; OB_SUCC(ret) && start != end; ++start) { + const ObRowkey &constraint_rowkey = start->first; + const ObConflictValue &constraint_value = start->second; + if (!is_row_changed_) { + // do nothing + } else if (constraint_value.new_row_source_ != ObNewRowSource::FROM_UPDATE) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected row source", K(ret), K(constraint_value.new_row_source_)); + } else { // FROM_UPDATE + // baseline_datum_row_ 代表存储扫描回来的冲突旧行 + // current_datum_row_ 当前更新的新行 + if (NULL != constraint_value.baseline_datum_row_ && + NULL != constraint_value.current_datum_row_) { + OZ(constraint_value.baseline_datum_row_->to_expr(get_primary_table_upd_old_row(), + eval_ctx_)); + OZ(delete_upd_old_row_to_das(constraint_rowkey, + constraint_value, + ttl_spec_.get_ctdef().upd_ctdef_, + ttl_rtdef_.upd_rtdef_, + upd_rtctx_)); + OZ(to_expr_skip_old(*constraint_value.current_datum_row_, + constraint_rowkey, + ttl_spec_.get_ctdef().upd_ctdef_)); + clear_evaluated_flag(); + OZ(insert_upd_new_row_to_das(ttl_spec_.get_ctdef().upd_ctdef_, + ttl_rtdef_.upd_rtdef_, + upd_rtctx_)); + } else if (NULL == constraint_value.baseline_datum_row_ && + NULL != constraint_value.current_datum_row_) { // 单单是唯一索引冲突的时候，会走这个分支 + OZ(to_expr_skip_old(*constraint_value.current_datum_row_, + constraint_rowkey, + ttl_spec_.get_ctdef().upd_ctdef_)); + OZ(insert_upd_new_row_to_das(ttl_spec_.get_ctdef().upd_ctdef_, + ttl_rtdef_.upd_rtdef_, + upd_rtctx_)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected constraint_value", K(ret)); + } + } + } + + return ret; +} + +int ObTableApiTTLExecutor::do_update() +{ + int ret = OB_SUCCESS; + const ObTableTTLCtDef &ctdef = ttl_spec_.get_ctdef(); + + // 1. 刷frame，刷到upd_ctdef.new_row中 + // 2. conflict_checker_.update_row + // 3. 遍历冲突map，do_update,参考ObTableApiInsertUpExecutor::do_update + + if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(1))) { // 1. 使用冲突行去回表，构造冲突行map，key为rowkey，value为旧行 + LOG_WARN("fail to do table lookup", K(ret)); + } else if (OB_FAIL(update_row_to_conflict_checker())) { // 2. 将更新的行刷到conflict_checker，目的是检查更新的行是否还有冲突 + LOG_WARN("fail to update row to conflict checker", K(ret)); + } else if (OB_FAIL(update_row_to_das())) { // 3. 根据实际情况更新行到das + LOG_WARN("fail to update row to das", K(ret)); + } + + return ret; +} + +// 1. 过期，删除旧行，写入新行 +// 2. 未过期 +// a. 当前是insert操作，则报错OB_ERR_ROWKEY_CONFLICT +// b. 当前是insertUp操作，则做update +int ObTableApiTTLExecutor::process_expire() +{ + int ret = OB_SUCCESS; + ObConflictRowMap *map = nullptr; + + if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(1))) { + LOG_WARN("fail to build conflict map", K(ret)); + } else if (OB_FAIL(conflict_checker_.get_primary_table_map(map))) { + LOG_WARN("fail to get conflict row map", K(ret)); + } else { + ObConflictRowMap::iterator start = map->begin(); + ObConflictRowMap::iterator end = map->end(); + for (; OB_SUCC(ret) && start != end; ++start) { + const ObRowkey &rowkey = start->first; + ObConflictValue &old_row = start->second; + if (NULL != old_row.baseline_datum_row_) { + // 将旧行刷到表达式中，判断是否过期 + if (OB_FAIL(old_row.baseline_datum_row_->to_expr(get_primary_table_upd_old_row(), eval_ctx_))) { + LOG_WARN("fail to change store row to expr", K(ret)); + } else if (OB_FAIL(check_expired(is_expired_))) { + LOG_WARN("fail to check expired", K(ret)); + } else if (is_expired_) { // 过期，删除旧行，写入新行 + LOG_DEBUG("row is expired", K(ret)); + if (OB_FAIL(do_delete())) { + LOG_WARN("fail to delete expired old row", K(ret)); + } else if (OB_FAIL(do_insert())) { + LOG_WARN("fail to insert new row", K(ret)); + } else if (OB_FAIL(execute_das_task(dml_rtctx_, true/* del_task_ahead */))) { + LOG_WARN("fail to execute insert das task", K(ret)); + } else { + insert_rows_ = 1; + } + } else { // 未过期, insert操作，则报错OB_ERR_ROWKEY_CONFLICT; insertUp操作，则做update + if (tb_ctx_.is_insert()) { + ret = OB_ERR_PRIMARY_KEY_DUPLICATE; + LOG_INFO("current insert is primary key duplicate", K(ret)); + } else { // insertUp + LOG_DEBUG("row not expired, do update", K(ret)); + if (OB_FAIL(do_update())) { + LOG_WARN("fail to do update", K(ret)); + } else if (OB_FAIL(execute_das_task(upd_rtctx_, true/* del_task_ahead */))) { + LOG_WARN("fail to execute upd_rtctx_ das task", K(ret)); + } else if (OB_FAIL(execute_das_task(dml_rtctx_, false/* del_task_ahead */))) { + LOG_WARN("fail to execute dml_rtctx_ das task", K(ret)); + } + } + } + } + } + } + + return ret; +} + +// 1. 获取快照点 +// 2. 写入记录 +// a. 不冲突，写入成功 +// b. 冲突，需要进一步判断是否是过期 +// ⅰ . 回归到快照点 +// ⅰⅰ. 处理过期逻辑 +// 1. 过期，删除旧行，写入新行 +// 2. 未过期 +// a. 当前是insert操作，则报错OB_ERR_ROWKEY_CONFLICT +// b. 当前是insertUp操作，则做update +int ObTableApiTTLExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + transaction::ObTxSEQ savepoint_no; + + set_need_fetch_conflict(upd_rtctx_, ttl_rtdef_.ins_rtdef_); + if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(exec_ctx_, savepoint_no))) { + LOG_WARN("fail to create save_point", K(ret)); + } else if (OB_FAIL(try_insert_row())) { + LOG_WARN("fail to do first insert", K(ret)); + } else if (!is_duplicated()) { + insert_rows_ = 1; + LOG_DEBUG("no duplicated, insert successfully"); + } else if (OB_FAIL(fetch_conflict_rowkey(conflict_checker_))) {// 获取所有的冲突主键 + LOG_WARN("fail to fetch conflict rowkey", K(ret)); + } else if (OB_FAIL(reset_das_env(ttl_rtdef_.ins_rtdef_))) { // reuse das 相关信息 + LOG_WARN("fail to reset das env", K(ret)); + } else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(exec_ctx_, savepoint_no))) { // 本次插入存在冲突, 回滚到save_point + LOG_WARN("fail to rollback to save_point", K(ret), K(savepoint_no)); + } else if (OB_FAIL(process_expire())) { // 处理过期 + LOG_WARN("fail to process expire", K(ret)); + } + + if (OB_SUCC(ret)) { + affected_rows_ = insert_rows_ + ttl_rtdef_.upd_rtdef_.found_rows_; + } + + return ret; +} + +int ObTableApiTTLExecutor::close() +{ + int ret = OB_SUCCESS; + int close_ret = OB_SUCCESS; + + if (is_opened_) { + if (OB_FAIL(conflict_checker_.close())) { + LOG_WARN("fail to close conflict_checker", K(ret)); + } + + if (upd_rtctx_.das_ref_.has_task()) { + close_ret = (upd_rtctx_.das_ref_.close_all_task()); + if (OB_SUCCESS == close_ret) { + upd_rtctx_.das_ref_.reset(); + } + } + ret = OB_SUCCESS == ret ? close_ret : ret; + // close dml das tasks + close_ret = ObTableApiModifyExecutor::close(); + } + + return (OB_SUCCESS == ret) ? close_ret : ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ttl/ob_table_ttl_executor.h b/src/observer/table/ttl/ob_table_ttl_executor.h new file mode 100644 index 0000000000..a6bb2e83dd --- /dev/null +++ b/src/observer/table/ttl/ob_table_ttl_executor.h @@ -0,0 +1,124 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_TTL_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_TTL_EXECUTOR_H +#include "src/observer/table/ob_table_modify_executor.h" +#include "src/observer/table/ob_table_context.h" + +namespace oceanbase +{ +namespace table +{ +class ObTableApiTTLSpec : public ObTableApiModifySpec +{ +public: + ObTableApiTTLSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + ttl_ctdef_(alloc), + conflict_checker_ctdef_(alloc) + { + } +public: + OB_INLINE const ObTableTTLCtDef& get_ctdef() const { return ttl_ctdef_; } + OB_INLINE ObTableTTLCtDef& get_ctdef() { return ttl_ctdef_; } + OB_INLINE const sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() const { return conflict_checker_ctdef_; } + OB_INLINE sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() { return conflict_checker_ctdef_; } +private: + ObTableTTLCtDef ttl_ctdef_; + sql::ObConflictCheckerCtdef conflict_checker_ctdef_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiTTLSpec); +}; + +class ObTableApiTTLExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiTTLExecutor(ObTableCtx &ctx, const ObTableApiTTLSpec &ttl_spec) + : ObTableApiModifyExecutor(ctx), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + ttl_spec_(ttl_spec), + upd_rtctx_(eval_ctx_, exec_ctx_, get_fake_modify_op()), + conflict_checker_(allocator_, eval_ctx_, ttl_spec_.get_conflict_checker_ctdef()), + cur_idx_(0), + is_row_changed_(false), + is_duplicated_(false), + is_expired_(false), + insert_rows_(0) + { + } + virtual ~ObTableApiTTLExecutor() + { + destroy(); + } +public: + virtual int open(); + virtual int get_next_row(); + virtual int close(); + virtual void destroy() + { + // destroy + conflict_checker_.destroy(); + upd_rtctx_.cleanup(); + ObTableApiModifyExecutor::destroy(); + } +public: + OB_INLINE bool is_insert_duplicated() + { + return is_duplicated_; + } + OB_INLINE bool is_expired() + { + return is_expired_; + } +private: + OB_INLINE bool is_duplicated() + { + is_duplicated_ = ttl_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_; + return is_duplicated_; + } + OB_INLINE const common::ObIArray &get_primary_table_insert_row() + { + return ttl_spec_.get_ctdef().ins_ctdef_.new_row_; + } + OB_INLINE const common::ObIArray &get_primary_table_upd_old_row() + { + return ttl_spec_.get_ctdef().upd_ctdef_.old_row_; + } + int check_expired(bool &is_expired); + int generate_ttl_rtdef(const ObTableTTLCtDef &ctdef, ObTableTTLRtDef &rtdef); + int get_next_row_from_child(); + int refresh_exprs_frame(const table::ObTableEntity *entity); + int do_insert(); + int do_delete(); + int try_insert_row(); + int process_expire(); + int update_row_to_conflict_checker(); + int update_row_to_das(); + int do_update(); +private: + common::ObArenaAllocator allocator_; + const ObTableApiTTLSpec &ttl_spec_; + ObTableTTLRtDef ttl_rtdef_; + sql::ObDMLRtCtx upd_rtctx_; + sql::ObConflictChecker conflict_checker_; + int64_t cur_idx_; + bool is_row_changed_; + bool is_duplicated_; + bool is_expired_; + int64_t insert_rows_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_TTL_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ttl/ob_table_ttl_task.cpp b/src/observer/table/ttl/ob_table_ttl_task.cpp new file mode 100644 index 0000000000..7051f6cdab --- /dev/null +++ b/src/observer/table/ttl/ob_table_ttl_task.cpp @@ -0,0 +1,626 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_ttl_task.h" +#include "share/table/ob_table_ttl_common.h" +#include "share/table/ob_table.h" +#include "observer/table/ob_table_op_wrapper.h" +#include "share/scheduler/ob_dag_warning_history_mgr.h" +#include "observer/table/ob_table_query_common.h" +#include "observer/table/ob_table_query_and_mutate_processor.h" +#include "lib/utility/utility.h" + +using namespace oceanbase::sql; +using namespace oceanbase::transaction; +using namespace oceanbase::observer; +using namespace oceanbase::storage; +using namespace oceanbase::share; +using namespace oceanbase::table; +using namespace oceanbase::rootserver; + + +/** + * ---------------------------------------- ObTableTTLDeleteTask ---------------------------------------- + */ +ObTableTTLDeleteTask::ObTableTTLDeleteTask(): + ObITask(ObITaskType::TASK_TYPE_TTL_DELETE), + is_inited_(false), + param_(), + info_(NULL), + allocator_(ObMemAttr(MTL_ID(), "TTLDeleteTask")), + rowkey_(), + ttl_tablet_mgr_(NULL), + default_entity_factory_("TTLEntityFac") +{ +} + +ObTableTTLDeleteTask::~ObTableTTLDeleteTask() +{ +} + +int ObTableTTLDeleteTask::init(ObTenantTabletTTLMgr *ttl_tablet_mgr, + const ObTTLTaskParam &ttl_para, + ObTTLTaskInfo &ttl_info) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_ISNULL(ttl_tablet_mgr) || ttl_info.table_id_ == OB_INVALID_ID || !ttl_info.ls_id_.is_valid() || !ttl_info.is_valid() || ttl_para.tenant_id_ == OB_INVALID_ID) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ttl_tablet_mgr), K(ttl_info), K(ttl_para)); + } else { + if (ttl_info.row_key_.empty()) { + rowkey_.reset(); + } else { + int64_t pos = 0; + if (OB_FAIL(rowkey_.deserialize(allocator_, ttl_info.row_key_.ptr(), ttl_info.row_key_.length(), pos))) { + LOG_WARN("fail to deserialize rowkey", KR(ret), K(ttl_info.row_key_)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(init_credential(ttl_para))) { + LOG_WARN("fail to init credential", KR(ret)); + } else if (OB_FAIL(create_session_pool(ttl_para.tenant_id_))) { + LOG_WARN("fail to update session pool"); + } else { + param_ = ttl_para; + info_ = &ttl_info; + ttl_tablet_mgr_ = ttl_tablet_mgr; + is_inited_ = true; + } + } + return ret; +} + +int ObTableTTLDeleteTask::create_session_pool(int64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(GCTX.table_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table service is null", KR(ret)); + } else if (OB_FAIL(GCTX.table_service_->get_sess_mgr().create_pool_if_not_exists(tenant_id))) { + LOG_WARN("fait to get session pool", K(ret), K(tenant_id)); + } + + return ret; +} + + +int ObTableTTLDeleteTask::init_credential(const ObTTLTaskParam &ttl_param) +{ + int ret = OB_SUCCESS; + const ObUserInfo *user_info = nullptr; + int64_t tenant_id = ttl_param.tenant_id_; + int64_t user_id = ttl_param.user_id_; + int64_t database_id = ttl_param.database_id_; + schema::ObSchemaGetterGuard schema_guard; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret), K(tenant_id)); + } else if(OB_FAIL(schema_guard.get_user_info(tenant_id, user_id, user_info))) { + LOG_WARN("fail to get user id", KR(ret), K(tenant_id), K(user_id)); + } else if (OB_ISNULL(user_info)) { + ret = OB_USER_NOT_EXIST; + LOG_WARN("user not exist", KR(ret), K(tenant_id), K(user_id)); + } else { + credential_.cluster_id_ = GCONF.cluster_id; + credential_.tenant_id_ = tenant_id; + credential_.user_id_ = user_id; + credential_.database_id_ = database_id; + credential_.expire_ts_ = 0; + credential_.hash(credential_.hash_val_, user_info->get_passwd_str().hash()); + } + + return ret; +} + + +int ObTableTTLDeleteTask::process() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + lib::ContextParam param; + param.set_mem_attr(MTL_ID(), "TTLDeleteTask", ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + CREATE_WITH_TEMP_CONTEXT(param) { + bool need_stop = false; + while(!need_stop) { + if (OB_FAIL(process_one())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to process one", KR(ret)); + } + } + if (OB_FAIL(ttl_tablet_mgr_->report_task_status(const_cast(*info_), param_, need_stop))) { + LOG_WARN("fail to report ttl task status", KR(ret)); + } + } + } + } + return ret; +} + +int ObTableTTLDeleteTask::process_one() +{ + int ret = OB_SUCCESS; + int64_t start_time = ObTimeUtil::current_time(); + ObTxDesc *trans_desc = nullptr; + ObTxReadSnapshot tx_snapshot; + TransState trans_state; + ObTableTTLOperationResult result; + ObTableApiSpec *scan_spec = nullptr; + observer::ObReqTimeGuard req_timeinfo_guard; // 引用cache资源必须加ObReqTimeGuard + ObTableApiCacheGuard cache_guard; + ObTableTTLOperation ttl_operation(info_->tenant_id_, + info_->table_id_, + param_, + PER_TASK_DEL_ROWS, + rowkey_); + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + if (OB_FAIL(init_scan_tb_ctx(tb_ctx, cache_guard))) { + LOG_WARN("fail to init tb ctx", KR(ret)); + } else if (OB_FAIL(ObTableApiProcessorBase::start_trans_( + false, + trans_desc, + tx_snapshot, + ObTableConsistencyLevel::STRONG, + &trans_state, + tb_ctx.get_table_id(), + tb_ctx.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("fail to start trans", KR(ret)); + } else if (OB_FAIL(tb_ctx.init_trans(trans_desc, tx_snapshot))) { + LOG_WARN("fail to init trans", KR(ret)); + } else if (OB_FAIL(cache_guard.get_spec(&tb_ctx, scan_spec))) { + LOG_WARN("fail to get scan spec from cache", KR(ret)); + } else { + ObTableTTLDeleteRowIterator row_iter; + ObTableApiExecutor *executor = nullptr; + if (OB_FAIL(scan_spec->create_executor(tb_ctx, executor))) { + LOG_WARN("fail to generate executor", KR(ret), K(tb_ctx)); + } else if (OB_FAIL(row_iter.init(*tb_ctx.get_table_schema(), ttl_operation))){ + LOG_WARN("fail to init ttl row iterator", KR(ret)); + } else if (OB_FAIL(row_iter.open(static_cast(executor)))) { + LOG_WARN("fail to open scan row iterator", KR(ret)); + } else if (OB_FAIL(execute_ttl_delete(row_iter, result, trans_desc, tx_snapshot))) { + LOG_WARN("fail to execute ttl table", KR(ret)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = row_iter.close())) { + LOG_WARN("fail to close row iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + + if (OB_NOT_NULL(scan_spec)) { + scan_spec->destroy_executor(executor); + tb_ctx.set_expr_info(nullptr); + } + } + } + + if (trans_state.is_start_trans_executed() && trans_state.is_start_trans_success()) { + int tmp_ret = ret; + if (OB_FAIL(ObTableApiProcessorBase::sync_end_trans_(OB_SUCCESS != ret, trans_desc, get_timeout_ts()))) { + LOG_WARN("fail to end trans", KR(ret)); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + } + + info_->max_version_del_cnt_ += result.get_max_version_del_row(); + info_->ttl_del_cnt_ += result.get_ttl_del_row(); + info_->scan_cnt_ += result.get_scan_row(); + info_->err_code_ = ret; + info_->row_key_ = result.get_end_rowkey(); + if (OB_SUCC(ret) && result.get_del_row() < PER_TASK_DEL_ROWS) { + ret = OB_ITER_END; // finsh task + info_->err_code_ = ret; + LOG_DEBUG("finish delete", KR(ret), KPC_(info)); + } + int64_t cost = ObTimeUtil::current_time() - start_time; + LOG_DEBUG("finish process one", KR(ret), K(cost)); + return ret; +} + +int ObTableTTLDeleteTask::init_scan_tb_ctx(ObTableCtx &tb_ctx, ObTableApiCacheGuard &cache_guard) +{ + int ret = OB_SUCCESS; + ObExprFrameInfo *expr_frame_info = nullptr; + tb_ctx.set_scan(true); + tb_ctx.set_operation_type(ObTableOperationType::DEL); + if (tb_ctx.is_init()) { + LOG_INFO("tb ctx has been inited", K(tb_ctx)); + } else if (OB_FAIL(tb_ctx.init_common(credential_, + get_tablet_id(), + get_table_id(), + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", KR(ret), "table_id", get_table_id(), + "tablet_id", get_tablet_id(), "timeout_ts", get_timeout_ts()); + } else if (OB_FAIL(tb_ctx.init_ttl_delete(get_start_rowkey()))) { + LOG_WARN("fail to init delete ctx", KR(ret), K(tb_ctx)); + } else if (OB_FAIL(cache_guard.init(&tb_ctx))) { + LOG_WARN("fail to init cache guard", K(ret)); + } else if (OB_FAIL(cache_guard.get_expr_info(&tb_ctx, expr_frame_info))) { + LOG_WARN("fail to get expr frame info from cache", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::alloc_exprs_memory(tb_ctx, *expr_frame_info))) { + LOG_WARN("fail to alloc expr memory", K(ret)); + } else if (OB_FAIL(tb_ctx.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", KR(ret), K(tb_ctx)); + } else { + tb_ctx.set_init_flag(true); + tb_ctx.set_expr_info(expr_frame_info); + } + + return ret; +} + +int ObTableTTLDeleteTask::process_ttl_delete(const ObITableEntity &new_entity, + int64_t &affected_rows, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot &snapshot) +{ + int ret = OB_SUCCESS; + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableOperationResult op_result; + if (OB_FAIL(init_tb_ctx(new_entity, tb_ctx))) { + LOG_WARN("fail to init table ctx", K(ret), K(new_entity)); + } else if (OB_FAIL(tb_ctx.init_trans(trans_desc, snapshot))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else if (OB_FAIL(ObTableOpWrapper::process_op(tb_ctx, op_result))) { + LOG_WARN("fail to process insert op", K(ret)); + } else { + affected_rows = op_result.get_affected_rows(); + } + } + return ret; +} + +int ObTableTTLDeleteTask::init_tb_ctx(const ObITableEntity &entity, + ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + ctx.set_entity(&entity); + ctx.set_entity_type(ObTableEntityType::ET_KV); + ctx.set_operation_type(ObTableOperationType::DEL); + ctx.set_batch_operation(NULL); + + if (!ctx.is_init()) { + if (OB_FAIL(ctx.init_common(credential_, + get_tablet_id(), + get_table_id(), + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), "table_id", get_table_id(), + "tablet_id", get_tablet_id(), "timeout_ts", get_timeout_ts()); + } else if (OB_FAIL(ctx.init_delete())) { + LOG_WARN("fail to init delete ctx", K(ret), K(ctx)); + } else if (OB_FAIL(ctx.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(ctx)); + } else { + ctx.set_is_ttl_table(false); + } + } + + return ret; +} + +/** + * ---------------------------------------- ObTableTTLDag ---------------------------------------- + */ +ObTableTTLDag::ObTableTTLDag() + : ObIDag(ObDagType::DAG_TYPE_TTL), + is_inited_(false), param_(), info_(), + compat_mode_(lib::Worker::CompatMode::INVALID) +{ +} + +ObTableTTLDag::~ObTableTTLDag() +{ +} + +int ObTableTTLDag::init(const ObTTLTaskParam ¶m, ObTTLTaskInfo &info) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTableTTLDag has already been inited", KR(ret)); + } else if (OB_UNLIKELY(!param.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid task para", KR(ret), K(info)); + } else if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid task info", KR(ret), K(param)); + } else if (OB_FAIL(ObCompatModeGetter::get_table_compat_mode(info.tenant_id_, info.table_id_, compat_mode_))) { + LOG_WARN("fail to get compat mode", KR(ret), K(info.tenant_id_), K(info.table_id_)); + } else { + param_ = param; + is_inited_ = true; + info_ = info; + consumer_group_id_ = info_.consumer_group_id_; + } + return ret; +} + +bool ObTableTTLDag::operator==(const ObIDag& other) const +{ + bool is_equal = false; + if (OB_UNLIKELY(this == &other)) { + is_equal = true; + } else if (get_type() == other.get_type()) { + const ObTableTTLDag &dag = static_cast(other); + if (OB_UNLIKELY(!param_.is_valid() || !dag.param_.is_valid() || + !dag.info_.is_valid() || !dag.info_.is_valid())) { + LOG_ERROR_RET(OB_ERR_SYS, "invalid argument", K_(param), K_(info), K(dag.param_), K(dag.info_)); + } else { + is_equal = (info_.tenant_id_ == dag.info_.tenant_id_) && + (info_.task_id_ == dag.info_.task_id_) && + (info_.table_id_ == dag.info_.table_id_) && + (info_.tablet_id_ == dag.info_.tablet_id_) && + (param_.ttl_ == dag.param_.ttl_) && + (param_.max_version_ == dag.param_.max_version_); + } + } + return is_equal; +} + +int64_t ObTableTTLDag::hash() const +{ + int64_t hash_val = 0; + if (OB_UNLIKELY(!is_inited_ || !param_.is_valid() || !info_.is_valid())) { + LOG_ERROR_RET(OB_ERR_SYS, "invalid argument", K(is_inited_), K(param_)); + } else { + hash_val = common::murmurhash(&info_.tenant_id_, sizeof(info_.tenant_id_), hash_val); + hash_val = common::murmurhash(&info_.task_id_, sizeof(info_.task_id_), hash_val); + hash_val = common::murmurhash(&info_.table_id_, sizeof(info_.table_id_), hash_val); + hash_val += info_.tablet_id_.hash(); + hash_val = common::murmurhash(¶m_.ttl_, sizeof(param_.ttl_), hash_val); + hash_val = common::murmurhash(¶m_.max_version_, sizeof(param_.max_version_), hash_val); + } + return hash_val; +} + +int ObTableTTLDag::fill_dag_key(char *buf, const int64_t buf_len) const +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableTTLDag has not been initialized", KR(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, "ttl task: ls_id = %ld, tenant_id = %ld, table_id = %ld, " + "tablet_id=%ld, max_version=%d, time_to_live=%d", + info_.ls_id_.id(), + info_.tenant_id_, + info_.table_id_, + info_.tablet_id_.id(), + param_.max_version_, + param_.ttl_))) { + LOG_WARN("fail to fill comment", KR(ret), K(param_), K(info_)); + } + return ret; +} + +int ObTableTTLDag::fill_info_param(compaction::ObIBasicInfoParam *&out_param, ObIAllocator &allocator) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(IS_NOT_INIT)) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableTTLDag has not been initialized", KR(ret)); + } else if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), + static_cast(info_.tenant_id_), + static_cast(info_.ls_id_.id()), + static_cast(info_.table_id_), + static_cast(info_.tablet_id_.id())))) { + LOG_WARN("fail to fill info param", KR(ret), K_(info)); + } + return ret; +} + +ObTableTTLDeleteRowIterator::ObTableTTLDeleteRowIterator() +: is_inited_(false), max_version_(0), time_to_live_ms_(0), limit_del_rows_(-1), cur_del_rows_(0), + cur_version_(0), cur_rowkey_(), cur_qualifier_(), max_version_cnt_(0), ttl_cnt_(0), scan_cnt_(0), + is_last_row_ttl_(true), is_hbase_table_(false), last_row_(nullptr), rowkey_cnt_(0) +{ +} + +int ObTableTTLDeleteRowIterator::init(const schema::ObTableSchema &table_schema, + const ObTableTTLOperation &ttl_operation) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("the table api ttl delete row iterator has been inited, ", KR(ret)); + } else if (OB_UNLIKELY(!ttl_operation.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid ttl operation", KR(ret), K(ttl_operation)); + } else if (OB_FAIL(ttl_checker_.init(table_schema, true))) { + LOG_WARN("fail to init ttl checker", KR(ret)); + } else { + time_to_live_ms_ = ttl_operation.time_to_live_ * 1000l; + max_version_ = ttl_operation.max_version_; + limit_del_rows_ = ttl_operation.del_row_limit_; + is_hbase_table_ = ttl_operation.is_htable_; + rowkey_cnt_ = table_schema.get_rowkey_column_num(); + ObSArray rowkey_column_ids; + ObSArray full_column_ids; + if (OB_FAIL(table_schema.get_rowkey_column_ids(rowkey_column_ids))) { + LOG_WARN("fail to get rowkey column ids", KR(ret)); + } else if (OB_FAIL(table_schema.get_column_ids(full_column_ids))) { + LOG_WARN("fail to get full column ids", KR(ret)); + } else { + for (int64_t i = 0, idx = -1; OB_SUCC(ret) && i < rowkey_column_ids.count(); i++) { + if (has_exist_in_array(full_column_ids, rowkey_column_ids[i], &idx)) { + if (OB_FAIL(rowkey_cell_ids_.push_back(idx))) { + LOG_WARN("fail to add rowkey cell idx", K(ret), K(i), K(rowkey_column_ids)); + } + } + } + } + + if (OB_SUCC(ret) && is_hbase_table_ && ttl_operation.start_rowkey_.is_valid()) { + if (ttl_operation.start_rowkey_.get_obj_cnt() != 3 || OB_ISNULL(ttl_operation.start_rowkey_.get_obj_ptr())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ttl_operation.start_rowkey_)); + } else { + ObObj *obj_ptr = const_cast(ttl_operation.start_rowkey_.get_obj_ptr()); + cur_rowkey_ = obj_ptr[ObHTableConstants::COL_IDX_K].get_string(); + cur_qualifier_ = obj_ptr[ObHTableConstants::COL_IDX_Q].get_string(); + is_inited_ = true; + } + } + } + return ret; +} + +// get next expired row to delete +int ObTableTTLDeleteRowIterator::get_next_row(ObNewRow*& row) +{ + int ret = OB_SUCCESS; + row = nullptr; + if (cur_del_rows_ >= limit_del_rows_) { + ret = OB_ITER_END; + LOG_DEBUG("finish get next row", KR(ret), K(cur_del_rows_), K(limit_del_rows_)); + } else { + bool is_expired = false; + while(OB_SUCC(ret) && !is_expired && OB_SUCC(ObTableApiScanRowIterator::get_next_row(row))) { + last_row_ = row; + // NOTE: For hbase table, the row expired if and only if + // 1. The row's version exceed maxversion + // 2. The row's expired time(cell_ts + ttl) exceed current time + if (is_hbase_table_) { + scan_cnt_++; + ObHTableCellEntity cell(row); + ObString cell_rowkey = cell.get_rowkey(); + ObString cell_qualifier = cell.get_qualifier(); + int64_t cell_ts = -cell.get_timestamp(); // obhtable timestamp is nagative in ms + if ((cell_rowkey != cur_rowkey_) || (cell_qualifier != cur_qualifier_)) { + cur_version_ = 1; + cur_rowkey_ = cell_rowkey; + cur_qualifier_ = cell_qualifier; + } else { + cur_version_++; + } + if (max_version_ > 0 && cur_version_ > max_version_) { + max_version_cnt_++; + cur_del_rows_++; + is_last_row_ttl_ = false; + is_expired = true; + } else if (time_to_live_ms_ > 0 && (cell_ts + time_to_live_ms_ < ObHTableUtils::current_time_millis())) { + ttl_cnt_++; + cur_del_rows_++; + is_last_row_ttl_ = true; + is_expired = true; + } + } else { + // NOTE: For relation table, the row expired if and only if + // 1. The row's expired time (the result of ttl definition) exceed current time + scan_cnt_++; + if (OB_FAIL(ttl_checker_.check_row_expired(*row, is_expired))) { + LOG_WARN("fail to check row expired", KR(ret)); + } else if (is_expired) { + ttl_cnt_++; + cur_del_rows_++; + is_last_row_ttl_ = true; + } + } + } + } + + if (OB_FAIL(ret)) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next row", KR(ret)); + } + } + return ret; +} + +int ObTableTTLDeleteTask::execute_ttl_delete(ObTableTTLDeleteRowIterator &ttl_row_iter, + ObTableTTLOperationResult &result, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot &snapshot) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + while (OB_SUCC(ret)) { + ObNewRow *row = nullptr; + ObITableEntity *new_entity = nullptr; + if (OB_FAIL(ttl_row_iter.get_next_row(row))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } else if (OB_ISNULL(new_entity = default_entity_factory_.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc entity", K(ret)); + } else { + int64_t rowkey_cnt = ttl_row_iter.rowkey_cell_ids_.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; i++) { + if (OB_FAIL(new_entity->add_rowkey_value(row->get_cell(ttl_row_iter.rowkey_cell_ids_[i])))) { + LOG_WARN("fail to add rowkey value", K(ret)); + } + } + + if (OB_SUCC(ret)) { + int64_t tmp_affect_rows = 0; + if (OB_FAIL(process_ttl_delete(*new_entity, tmp_affect_rows, trans_desc, snapshot))) { + LOG_WARN("fail to execute table delete", K(ret)); + } else { + affected_rows += tmp_affect_rows; + } + } + } + } + + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + + if (OB_SUCC(ret)) { + uint64_t iter_ttl_cnt = ttl_row_iter.ttl_cnt_; + uint64_t iter_max_version_cnt = ttl_row_iter.max_version_cnt_; + uint64_t iter_return_cnt = iter_ttl_cnt + iter_max_version_cnt; + if (OB_UNLIKELY(affected_rows != iter_return_cnt)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected affect rows", K(ret), K(affected_rows), K(iter_return_cnt)); + } else { + result.ttl_del_rows_ = iter_ttl_cnt; + result.max_version_del_rows_ = iter_max_version_cnt; + result.scan_rows_ = ttl_row_iter.scan_cnt_; + } + } + if (OB_SUCC(ret)) { + ObRowkey row_key; + if (OB_NOT_NULL(ttl_row_iter.last_row_)) { + if (ttl_row_iter.last_row_->get_count() < ttl_row_iter.get_rowkey_column_cnt()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rowkey column count", K(ttl_row_iter.last_row_->get_count()), K(ttl_row_iter.get_rowkey_column_cnt())); + } else { + row_key.assign(ttl_row_iter.last_row_->cells_, ttl_row_iter.get_rowkey_column_cnt()); + } + } + + if (OB_SUCC(ret) && row_key.is_valid()) { + uint64_t buf_len = row_key.get_serialize_size(); + char *buf = static_cast(allocator_.alloc(buf_len)); + int64_t pos = 0; + if (OB_FAIL(row_key.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", K(ret), K(buf_len), K(pos), K(row_key)); + } else { + result.end_rowkey_.assign_ptr(buf, buf_len); + } + } + } + LOG_DEBUG("execute ttl delete", K(ret), K(result)); + return ret; +} diff --git a/src/observer/table/ttl/ob_table_ttl_task.h b/src/observer/table/ttl/ob_table_ttl_task.h new file mode 100644 index 0000000000..16ab240e36 --- /dev/null +++ b/src/observer/table/ttl/ob_table_ttl_task.h @@ -0,0 +1,143 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_TTL_TASK_H_ +#define OCEANBASE_OBSERVER_OB_TABLE_TTL_TASK_H_ +#include "share/table/ob_table_ttl_common.h" +#include "sql/ob_sql_trans_control.h" +#include "share/scheduler/ob_dag_scheduler.h" +#include "observer/table/ob_table_context.h" +#include "observer/table/ob_table_scan_executor.h" +#include "share/table/ob_table.h" +#include "observer/table/ob_table_cache.h" +#include "observer/table/ttl/ob_tenant_tablet_ttl_mgr.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTableTTLDeleteRowIterator : public table::ObTableApiScanRowIterator +{ +public: + ObTableTTLDeleteRowIterator(); + ~ObTableTTLDeleteRowIterator() {} + virtual int get_next_row(ObNewRow*& row); + int init(const share::schema::ObTableSchema &table_schema, const table::ObTableTTLOperation &ttl_operation); + int64_t get_rowkey_column_cnt() const { return rowkey_cnt_; } +public: + bool is_inited_; + int32_t max_version_; + int64_t time_to_live_ms_; // ttl in millisecond + uint64_t limit_del_rows_; // maximum delete row + uint64_t cur_del_rows_; // current delete row + uint64_t cur_version_; + ObString cur_rowkey_; // K + ObString cur_qualifier_; // Q + uint64_t max_version_cnt_; + uint64_t ttl_cnt_; + uint64_t scan_cnt_; + bool is_last_row_ttl_; // false indicate row del by max version + bool is_hbase_table_; + ObNewRow *last_row_; + common::ObTableTTLChecker ttl_checker_; + int64_t rowkey_cnt_; + + common::ObSArray rowkey_cell_ids_; +}; + + +class ObTableTTLDeleteTask : public share::ObITask +{ +public: + ObTableTTLDeleteTask(); + ~ObTableTTLDeleteTask(); + int init(table::ObTenantTabletTTLMgr *ttl_tablet_mgr, + const table::ObTTLTaskParam &ttl_para, + table::ObTTLTaskInfo &ttl_info); + int init_tb_ctx(const table::ObITableEntity &entity, table::ObTableCtx &ctx); + int init_scan_tb_ctx(table::ObTableCtx &tb_ctx, table::ObTableApiCacheGuard &cache_guard); + int execute_ttl_delete(ObTableTTLDeleteRowIterator &ttl_row_iter, + table::ObTableTTLOperationResult &result, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot &snapshot); + int process_ttl_delete(const table::ObITableEntity &new_entity, + int64_t &affected_rows, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot &snapshot); + table::ObTableEntityFactory &get_entity_factory() { return default_entity_factory_; } + common::ObIAllocator &get_allocator() { return allocator_; } + int init_credential(const table::ObTTLTaskParam &ttl_param); + + virtual int process() override; + common::ObTabletID get_tablet_id() const + { + common::ObTabletID tablet_id; + if (OB_NOT_NULL(info_)) { + tablet_id = info_->get_tablet_id(); + } + return tablet_id; + } + int create_session_pool(int64_t tenant_id); + uint64_t get_table_id() const { return param_.table_id_; } + int64_t get_timeout_ts() { return ONE_TASK_TIMEOUT + ObTimeUtility::current_time(); } + common::ObRowkey &get_start_rowkey() { return rowkey_; } + +private: + static const int64_t RETRY_INTERVAL = 30 * 60 * 1000 * 1000l; // 30min + static const int64_t PER_TASK_DEL_ROWS = 10000l; + static const int64_t ONE_TASK_TIMEOUT = 1 * 60 * 1000 * 1000l; // 1min + +private: + int process_one(); + +private: + bool is_inited_; + table::ObTTLTaskParam param_; + table::ObTTLTaskInfo *info_; + common::ObArenaAllocator allocator_; + common::ObRowkey rowkey_; + table::ObTenantTabletTTLMgr *ttl_tablet_mgr_; + share::ObLSID ls_id_; + table::ObTableEntityFactory default_entity_factory_; + table::ObTableApiCredential credential_; + DISALLOW_COPY_AND_ASSIGN(ObTableTTLDeleteTask); +}; + +class ObTableTTLDag final: public share::ObIDag +{ +public: + ObTableTTLDag(); + virtual ~ObTableTTLDag(); + virtual bool operator==(const ObIDag& other) const override; + virtual int64_t hash() const override; + int init(const table::ObTTLTaskParam ¶m, table::ObTTLTaskInfo &info); + virtual lib::Worker::CompatMode get_compat_mode() const override { return compat_mode_; } + virtual int fill_dag_key(char *buf, const int64_t buf_len) const override; + virtual int fill_info_param(compaction::ObIBasicInfoParam *&out_param, ObIAllocator &allocator) const override; + virtual uint64_t get_consumer_group_id() const override { return consumer_group_id_; } + + virtual bool is_ha_dag() const { return false; } + +private: + bool is_inited_; + table::ObTTLTaskParam param_; + table::ObTTLTaskInfo info_; + lib::Worker::CompatMode compat_mode_; + + DISALLOW_COPY_AND_ASSIGN(ObTableTTLDag); +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_TTL_TASK_H_ */ diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp new file mode 100644 index 0000000000..d80aae7e99 --- /dev/null +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp @@ -0,0 +1,1373 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_tenant_tablet_ttl_mgr.h" +#include "share/ob_srv_rpc_proxy.h" +#include "share/schema/ob_multi_version_schema_service.h" +#include "observer/table/ob_htable_filter_operator.h" +#include "observer/table/ob_htable_utils.h" +#include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_tenant_freezer.h" +#include "observer/table/ttl/ob_table_ttl_task.h" +#include "observer/table/ob_table_service.h" + +namespace oceanbase +{ + +using namespace storage; +using namespace obrpc; +using namespace share; +using namespace table; +using namespace observer; + +namespace table +{ + +int ObTenantTabletTTLMgr::init(ObLS *ls) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get sql proxy from GCTX", KR(ret)); + } else if (OB_ISNULL(ls)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get ls", KR(ret)); + } else if (OB_FAIL(init(MTL_ID()))) { + LOG_WARN("fail to init tenant ttl mgr", KR(ret), K(MTL_ID())); + } else { + ls_ = ls; + sql_proxy_ = GCTX.sql_proxy_; + LOG_INFO("success to init tenant ttl mgr", K(MTL_ID())); + } + return ret; +} + +int ObTenantTabletTTLMgr::init(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + schema_service_ = &(schema::ObMultiVersionSchemaService::get_instance()); + + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(schema_service_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("schema service is null", KR(ret)); + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TenantTabletTTLMgr, tg_id_))) { + LOG_WARN("fail to init timer", KR(ret)); + } else if (OB_FAIL(alloc_tenant_info(tenant_id))) { + LOG_WARN("fail to alloc tenant info", KR(ret), K(MTL_ID())); + } else { + tablet_table_pairs_.set_attr(ObMemAttr(tenant_id, "TTLTablePairs")); + tenant_id_ = tenant_id; + is_inited_ = true; + } + FLOG_INFO("finish init tenant_tablet_ttl_mgr", K_(tenant_id), K(ret)); + return ret; +} + +int ObTenantTabletTTLMgr::switch_to_leader() +{ + int64_t start_time_us = ObTimeUtility::current_time(); + FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_leader", K_(tenant_id), K(start_time_us)); + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("fail to check_inner_stat", KR(ret), K_(tenant_id)); + } else if (!has_start_) { + if (OB_FAIL(start())) { + LOG_WARN("fail to start tablet ttl mgr", K(ret)); + } else { + has_start_ = true; + } + } else { + resume(); + } + const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; + FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_leader", KR(ret), K_(tenant_id), K(cost_us)); + + return ret; +} + +int ObTenantTabletTTLMgr::check_inner_stat() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K_(tenant_id)); + } + return ret; +} + +int ObTenantTabletTTLMgr::switch_to_follower_gracefully() +{ + int ret = OB_SUCCESS; + inner_switch_to_follower(); + return ret; +} + +void ObTenantTabletTTLMgr::switch_to_follower_forcedly() +{ + inner_switch_to_follower(); +} + +void ObTenantTabletTTLMgr::inner_switch_to_follower() +{ + FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_follower", K_(tenant_id)); + const int64_t start_time_us = ObTimeUtility::current_time(); + pause(); + const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; + FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_follower", K_(tenant_id), K(cost_us)); +} + +void ObTenantTabletTTLMgr::resume() +{ + is_paused_ = false; +} + +void ObTenantTabletTTLMgr::pause() +{ + is_paused_ = true; +} + +int ObTenantTabletTTLMgr::start() +{ + int ret = OB_SUCCESS; + FLOG_INFO("tenant_tablet_ttl_mgr: begin to start", K_(tenant_id)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else if (OB_FAIL(TG_START(tg_id_))) { + LOG_WARN("failed to create ObTenantTabletTTLMgr thread", K(ret), K_(tg_id)); + } else if (OB_FAIL(TG_SCHEDULE(tg_id_, periodic_task_, periodic_delay_, true))) { + LOG_WARN("fail to schedule periodic task", KR(ret), K_(tg_id)); + } else { + is_timer_start_ = true; + } + FLOG_INFO("tenant_tablet_ttl_mgr: finish to start", K(ret), K_(tenant_id)); + return ret; +} + +void ObTenantTabletTTLMgr::stop() +{ + int ret = OB_SUCCESS; + FLOG_INFO("tenant_tablet_ttl_mgr: begin to stop", K_(tenant_id)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else if (is_timer_start_) { + TG_STOP(tg_id_); + is_timer_start_ = false; + } + FLOG_INFO("tenant_tablet_ttl_mgr: finish to stop", K(ret), K_(tenant_id)); +} + +void ObTenantTabletTTLMgr::wait() +{ + FLOG_INFO("tenant_tablet_ttl_mgr: begin to wait", K_(tenant_id)); + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else { + TG_WAIT(tg_id_); + } + FLOG_INFO("tenant_tablet_ttl_mgr: finish to wait", K_(tenant_id), K(ret)); +} + +void ObTenantTabletTTLMgr::destroy() +{ + int ret = OB_SUCCESS; + FLOG_INFO("tenant_tablet_ttl_mgr: begin to destroy", K_(tenant_id)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else { + TG_DESTROY(tg_id_); + is_inited_ = false; + } + FLOG_INFO("tenant_tablet_ttl_mgr: finish to destroy", K_(tenant_id), K(ret)); +} + +int ObTenantTabletTTLMgr::check_and_handle_event() +{ + int ret = OB_SUCCESS; + bool need_check = false; + bool is_dirty = false; + bool is_finished = false; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl manager not init", KR(ret)); + } else if (is_paused_) { + // do nothing, not leader + } else { + if (OB_FAIL(check_schema_version())) { + LOG_WARN("fail to check schema version", KR(ret)); + } else { + common::ObSpinLockGuard guard(lock_); + // after observer restart, need check tenant even when cancel and move state + is_dirty = local_tenant_task_.is_dirty_; + is_finished = local_tenant_task_.is_finished_; + need_check = !is_finished && local_tenant_task_.need_check_; + } + + if (OB_SUCC(ret) && need_check) { + if (OB_FAIL(check_and_generate_tablet_tasks())) { + LOG_WARN("fail to generate tablet tasks", K_(tenant_id)); + } + } + + if (OB_SUCC(ret) && (is_dirty || !is_finished)) { + common::ObSArray dirty_tasks; + if (OB_FAIL(handle_all_tablet_event(dirty_tasks))) { + LOG_WARN("fail to handle tenant event in timer", K_(tenant_id)); + } + + for (int i = 0; OB_SUCC(ret) && i < dirty_tasks.count(); i++) { + if (OB_FAIL(sync_sys_table(dirty_tasks.at(i)))) { + LOG_WARN("fail to sync sys table", KR(ret)); + } + } + + check_ttl_tenant_state(); // check whether tenant still dirty or not + } + } + + return ret; +} + +void ObTenantTabletTTLMgr::check_ttl_tenant_state() +{ + common::ObSpinLockGuard guard(lock_); + int ret = OB_SUCCESS; + bool tenant_dirty = false; + bool tenant_finish = true; + ObTTLTaskCtx* ctx = nullptr; + + for (tablet_task_iter iter = local_tenant_task_.tablet_task_map_.begin(); + !tenant_dirty && iter != local_tenant_task_.tablet_task_map_.end(); ++iter) { + ctx = iter->second; + if (OB_ISNULL(ctx)) { + LOG_WARN("fatal err, ttl ctx in map is null", K(local_tenant_task_.tenant_id_)); + } else if (ctx->is_dirty_) { + tenant_dirty = true; + } else if (ctx->task_status_ != OB_TTL_TASK_CANCEL && + ctx->task_status_ != OB_TTL_TASK_FINISH) { + tenant_finish = false; + local_tenant_task_.is_finished_ = false; + } + } + if (OB_SUCC(ret) && !tenant_dirty) { + local_tenant_task_.is_dirty_ = false; + if (tenant_finish) { + // all task already in cancel or runing status + if (local_tenant_task_.state_ == OB_TTL_TASK_CANCEL || local_tenant_task_.state_ == OB_TTL_TASK_RUNNING) { + local_tenant_task_.is_finished_ = true; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tenant ttl state", KR(ret), K(local_tenant_task_.state_)); + } + } + } + + if (OB_SUCC(ret) && local_tenant_task_.is_finished_) { + FLOG_INFO("local ls ttl task is finished", K_(local_tenant_task), KPC_(ls)); + local_tenant_task_.reuse(); + } + + LOG_DEBUG("check ttl tenant dirty", K(local_tenant_task_.is_dirty_), K(local_tenant_task_.state_), KR(ret), K_(tenant_id)); +} + +int ObTenantTabletTTLMgr::check_cmd_state_valid(const common::ObTTLTaskStatus current_state, + const common::ObTTLTaskStatus incoming_state) +{ + int ret = OB_SUCCESS; + switch (incoming_state) { + case OB_TTL_TASK_RUNNING: { + if (current_state != OB_TTL_TASK_PENDING && current_state != OB_TTL_TASK_INVALID && + current_state != OB_TTL_TASK_RUNNING) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("receive rs cmd, but current tenant state is unmatached", + KR(ret), K(current_state), K(incoming_state)); + } + break; + } + case OB_TTL_TASK_MOVING: { + if (current_state != OB_TTL_TASK_RUNNING && current_state != OB_TTL_TASK_CANCEL && + current_state != OB_TTL_TASK_INVALID && current_state != OB_TTL_TASK_MOVING) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("receive a move cmd, current task state is unmatached", K(current_state)); + } + break; + } + case OB_TTL_TASK_PENDING: { + if (current_state != OB_TTL_TASK_RUNNING && current_state != OB_TTL_TASK_INVALID && + current_state != OB_TTL_TASK_PENDING) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("receive rs cmd, but current tenant state is unmatached", + KR(ret), K(current_state), K(incoming_state)); + } + break; + } + case OB_TTL_TASK_CANCEL: { + if (current_state != OB_TTL_TASK_PENDING && current_state != OB_TTL_TASK_RUNNING && + current_state != OB_TTL_TASK_INVALID && current_state != OB_TTL_TASK_CANCEL) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("receive rs cmd, but current tenant state is unmatached", + KR(ret), K(current_state), K(incoming_state)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid incoming status", KR(ret), K(incoming_state)); + break; + } + } + return ret; +} + +int ObTenantTabletTTLMgr::transform_tenant_state(const common::ObTTLTaskStatus& tenant_status, + common::ObTTLTaskStatus& status) +{ + int ret = OB_SUCCESS; + if (tenant_status == OB_RS_TTL_TASK_CREATE) { + status = OB_TTL_TASK_RUNNING; + } else if (tenant_status == OB_RS_TTL_TASK_SUSPEND) { + status = OB_TTL_TASK_PENDING; + } else if (tenant_status == OB_RS_TTL_TASK_CANCEL) { + status = OB_TTL_TASK_CANCEL; + } else if (tenant_status == OB_RS_TTL_TASK_MOVE) { + status = OB_TTL_TASK_MOVING; + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid type", K(tenant_status), K(status)); + } + return ret; +} + +void ObTenantTabletTTLMgr::mark_tenant_need_check() +{ + int ret = OB_SUCCESS; + if (common::ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + common::ObSpinLockGuard guard(lock_); + local_tenant_task_.need_check_ = true; + } + LOG_DEBUG("finsh mark tenant need check", KR(ret)); +} + +void ObTenantTabletTTLMgr::on_schema_changed(uint64_t schema_changed_tenant_id) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl manager not init"); + } else if (!common::ObTTLUtil::check_can_process_tenant_tasks(schema_changed_tenant_id)) { + //do nothing + } else { + mark_tenant_need_check(); + } +} + +int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTaskParam& task_para, bool& is_stop) +{ + int ret = OB_SUCCESS; + ObTTLTaskCtx* ctx = nullptr; + bool task_need_retry = false; + common::ObSpinLockGuard guard(lock_); + is_stop = true; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr is not init,", KR(ret)); + } else if (OB_FAIL(local_tenant_task_.tablet_task_map_.get_refactored(task_info.tablet_id_, ctx))) { + LOG_WARN("fail to get tablet task", KR(ret), K_(tenant_id)); + } else if (OB_ISNULL(ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the tablet task ctx is null", KR(ret)); + } else { + // lock task ctx for update + common::ObSpinLockGuard ctx_guard(ctx->lock_); + ctx->last_modify_time_ = ObTimeUtility::current_time(); + mark_ttl_ctx_dirty(local_tenant_task_, *ctx); + if (OB_FAIL(deep_copy_task(ctx, task_info, task_para))) { + LOG_WARN("fail to refresh ttl ctx content", KR(ret), K_(tenant_id)); + } else if (OB_SUCCESS == task_info.err_code_) { + if (local_tenant_task_.ttl_continue_ && OB_TTL_TASK_RUNNING == local_tenant_task_.state_) { + task_para = ctx->ttl_para_; + is_stop = false; + } else { + LOG_INFO("pending current task", K(local_tenant_task_.state_), K(local_tenant_task_.ttl_continue_)); + } + } else if (OB_ITER_END == task_info.err_code_) { + ctx->task_status_ = OB_TTL_TASK_FINISH; + ctx->task_info_.err_code_ = OB_SUCCESS; + } else if (OB_NOT_MASTER == task_info.err_code_ || + OB_PARTITION_NOT_EXIST == task_info.err_code_ || + OB_TABLE_NOT_EXIST == task_info.err_code_ || + OB_ERR_UNKNOWN_TABLE == task_info.err_code_) { + LOG_INFO("cancel current task since partition state change", + K(task_info.err_code_), K(task_info.tablet_id_)); + ctx->task_status_ = OB_TTL_TASK_CANCEL; + ctx->is_invalid_ = true; + } else { + LOG_WARN("task report error", K(task_info.err_code_), K(task_info.tablet_id_)); + ctx->task_status_ = OB_TTL_TASK_PENDING; + ctx->failure_times_++; + } + } + + //schedule task + if (is_stop && OB_FAIL(try_schedule_remaining_tasks(ctx))) { + LOG_WARN("fail to try schedule task", KR(ret)); + } + return ret; +} + +int ObTenantTabletTTLMgr::generate_batch_tablet_task(ObIArray& tablet_pairs, + hash::ObHashMap ¶m_map) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_pairs.count(); i++) { + ObTTLTaskInfo task_info; + uint64_t table_id = tablet_pairs.at(i).get_table_id(); + task_info.tablet_id_ = tablet_pairs.at(i).get_tablet_id(); + task_info.tenant_id_ = tenant_id_; + task_info.table_id_ = table_id; + ObTTLTaskParam param; + if (OB_FAIL(param_map.get_refactored(table_id, param))) { + LOG_WARN("fail to get ttl param", KR(ret), K(table_id)); + } else if (OB_FAIL(generate_one_tablet_task(task_info, param))) { + LOG_WARN("fail to generate task", KR(ret), K(task_info), K(param)); + } + } + + FLOG_INFO("finish generate tablet task", KR(ret), K(tablet_pairs)); + return ret; +} + +int ObTenantTabletTTLMgr::generate_one_tablet_task(ObTTLTaskInfo& task_info, const ObTTLTaskParam& param) +{ + int ret = OB_SUCCESS; + ObTTLTaskCtx* ctx = nullptr; + common::ObSpinLockGuard guard(lock_); + int64_t task_time = ObTimeUtility::current_time(); + + if (OB_NOT_NULL(ctx = get_one_tablet_ctx(task_info.tablet_id_))) { + LOG_INFO("ttl ctx exist", KR(ret), K(task_info.tablet_id_)); + } else { + char *ctx_buf = static_cast(local_tenant_task_.allocator_.alloc(sizeof(ObTTLTaskCtx))); + if (OB_ISNULL(ctx_buf)) { + LOG_WARN("fail to alloc ttl task ctx", KR(ret)); + } else { + ctx = new(ctx_buf)ObTTLTaskCtx(); + ctx->task_status_ = OB_TTL_TASK_PREPARE; + ctx->ttl_para_ = param; + mark_ttl_ctx_dirty(local_tenant_task_, *ctx); + ctx->task_start_time_ = task_time; + ctx->last_modify_time_ = task_time; + ctx->task_info_.tablet_id_ = task_info.tablet_id_; + ctx->task_info_.is_user_trigger_ = local_tenant_task_.is_usr_trigger_; + ctx->task_info_.task_id_ = local_tenant_task_.task_id_; + ctx->task_info_.ls_id_ = ls_->get_ls_id(); + ctx->task_info_.tenant_id_ = task_info.tenant_id_; + ctx->task_info_.table_id_ = task_info.table_id_; + if (OB_FAIL(deep_copy_task(ctx, task_info, param))) { + LOG_WARN("fail tp deep copy task", KR(ret)); + } else { + if (OB_FAIL(local_tenant_task_.tablet_task_map_.set_refactored(task_info.tablet_id_, ctx))) { + LOG_WARN("fail to insert ttl task ctx into map", KR(ret), K(task_info.tablet_id_)); + local_tenant_task_.allocator_.free(ctx); + ctx = nullptr; + } + } + } + } + LOG_DEBUG("finish generate one partition task", KR(ret), K(task_info.tablet_id_), K(param)); + return ret; +} + +void ObTenantTabletTTLMgr::mark_tenant_checked() +{ + common::ObSpinLockGuard guard(lock_); + local_tenant_task_.need_check_ = false; +} + +// scan all ttl tablet in current LS, and generate ttl dag task if necessary +int ObTenantTabletTTLMgr::check_and_generate_tablet_tasks() +{ + int ret = OB_SUCCESS; + bool can_ttl = false; + ObTabletHandle tablet_handle; + ObSEArray table_schema_arr; + const schema::ObTableSchema *table_schema = nullptr; + ObSEArray table_id_array; + hash::ObHashMap param_map; // table_id, task param + ObMemAttr bucket_attr(tenant_id_, "TTLTaskParBuck"); + ObMemAttr node_attr(tenant_id_, "TTLTasParkNode"); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::get_tenant_table_ids(tenant_id_, table_id_array))) { + LOG_WARN("fail to get tenant table ids", KR(ret), K_(tenant_id)); + } else if (!table_id_array.empty() && OB_FAIL(tablet_table_pairs_.reserve(DEFAULT_TABLET_PAIR_SIZE))) { + LOG_WARN("fail to reserve", KR(ret)); + } else if (!table_id_array.empty() && OB_FAIL(param_map.create(DEFAULT_PARAM_BUCKET_SIZE, bucket_attr, node_attr))) { + LOG_WARN("fail to create param map", KR(ret)); + } + + int64_t start_idx = 0; + int64_t end_idx = 0; + while (OB_SUCC(ret) && start_idx < table_id_array.count()) { + { + // temp schema guard to loop table ids + ObSchemaGetterGuard schema_guard; + start_idx = end_idx; + end_idx = MIN(table_id_array.count(), start_idx + TBALE_GENERATE_BATCH_SIZE); + bool is_ttl_table = false; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret), K_(tenant_id)); + } + for (int64_t idx = start_idx; OB_SUCC(ret) && idx < end_idx; ++idx) { + const int64_t table_id = table_id_array.at(idx); + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, table_id, table_schema))) { + LOG_WARN("failed to get simple schema", KR(ret), K(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table schema is null", KR(ret), K(table_id), K_(tenant_id)); + } else if (OB_FAIL(ObTTLUtil::check_is_ttl_table(*table_schema, is_ttl_table))) { + LOG_WARN("fail to check is ttl table", KR(ret)); + } else if (is_ttl_table) { + ObArray tablet_ids; + ObTTLTaskParam ttl_param; + if (OB_FAIL(table_schema->get_tablet_ids(tablet_ids))) { + LOG_WARN("fail to get tablet ids", KR(ret), K(table_id)); + } else if (OB_FAIL(get_ttl_para_from_schema(table_schema, ttl_param))) { + LOG_WARN("fail to get ttl para"); + } else if (OB_FAIL(param_map.set_refactored(table_id, ttl_param))) { + LOG_WARN("fail to push back table ttl param pairs", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { + if (OB_FAIL(ls_->get_tablet_svr()->get_tablet(tablet_ids.at(i), tablet_handle))) { + if (OB_TABLET_NOT_EXIST != ret) { + LOG_WARN("fail to get tablet", K(ret), K(tablet_ids.at(i))); + } else { + ret = OB_SUCCESS; + } + } else if (OB_NOT_NULL(get_one_tablet_ctx(tablet_ids.at(i)))) { + // do nothing + } else { + ObTabletTablePair cur_pair; + ObTabletID tablet_id = tablet_ids.at(i); + if (OB_FAIL(cur_pair.init(tablet_id, table_id))) { + LOG_WARN("fail to init tablet_ls_pair", KR(ret), K(i), K(tablet_id), K(table_id)); + } else if (OB_FAIL(tablet_table_pairs_.push_back(cur_pair))) { + LOG_WARN("fail to push back pair", KR(ret), K(cur_pair)); + } + } + } + } + } + } + } // end scope + if (OB_SUCC(ret) && OB_FAIL(generate_batch_tablet_task(tablet_table_pairs_, param_map))) { + LOG_WARN("fail to generate batch tablet task", KR(ret)); + } + tablet_table_pairs_.reuse(); + } + tablet_table_pairs_.reset(); + + if (OB_SUCC(ret)) { + mark_tenant_checked(); + } + LOG_DEBUG("finish generate tenant tasks", KR(ret), K_(tenant_id)); + return ret; +} + +void OBTTLTimerPeriodicTask::runTimerTask() +{ + int ret = OB_SUCCESS; + ObCurTraceId::init(GCONF.self_addr_); + if (common::ObTTLUtil::check_can_do_work()) { + if (OB_FAIL(tablet_ttl_mgr_.check_tenant_memory())) { + LOG_WARN("fail to check all tenant memory", KR(ret)); + } + + // explicit cover error code + ret = OB_SUCCESS; + if (OB_FAIL(tablet_ttl_mgr_.reload_tenant_task())) { + LOG_WARN("fail to reload tenant task", KR(ret)); + } + + // explicit cover error code + ret = OB_SUCCESS; + if (OB_FAIL(tablet_ttl_mgr_.check_and_handle_event())) { + LOG_WARN("fail to scan and handle all tenant event", KR(ret)); + } + } +} + +int ObTenantTabletTTLMgr::alloc_tenant_info(uint64_t tenant_id) +{ + + int ret = OB_SUCCESS; + local_tenant_task_.tenant_id_ = tenant_id; + local_tenant_task_.ttl_continue_ = false; + local_tenant_task_.is_dirty_ = true; + ObMemAttr bucket_attr(tenant_id, "TTLTaskBucket"); + ObMemAttr node_attr(tenant_id, "TTLTaskNode"); + if(OB_FAIL(local_tenant_task_.tablet_task_map_.create(DEFAULT_TTL_BUCKET_NUM, bucket_attr, node_attr))) { + LOG_WARN("fail to create ttl partition map", KR(ret), K(tenant_id)); + } else {} + + FLOG_INFO("finish create tenant ttl task", K(tenant_id), K(ret)); + return ret; +} + +ObTTLTaskCtx* ObTenantTabletTTLMgr::get_one_tablet_ctx(const ObTabletID& tablet_id) +{ + int ret = OB_SUCCESS; + ObTTLTaskCtx* ttl_task_ctx = nullptr; + + if (OB_FAIL(local_tenant_task_.tablet_task_map_.get_refactored(tablet_id, ttl_task_ctx))) { + LOG_DEBUG("fail to get partition task", KR(ret), K(tablet_id)); + } else if (OB_ISNULL(ttl_task_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the partition task ctx is null", KR(ret)); + } + return ttl_task_ctx; +} + +/*other inner function*/ +int ObTenantTabletTTLMgr::deep_copy_task(ObTTLTaskCtx* ctx, ObTTLTaskInfo& task_info, const ObTTLTaskParam &task_param) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the ctx is null", KR(ret)); + } else if (OB_FAIL(ob_write_string(allocator_, task_info.row_key_, ctx->task_info_.row_key_)) ) { + LOG_WARN("fail to deep copy rowkey", KR(ret), K(task_info.row_key_)); + } else { + ctx->task_info_.ttl_del_cnt_ = task_info.ttl_del_cnt_; + ctx->task_info_.max_version_del_cnt_ = task_info.max_version_del_cnt_; + ctx->task_info_.scan_cnt_ = task_info.scan_cnt_; + ctx->task_info_.err_code_ = task_info.err_code_; + } + return ret; +} + +int ObTenantTabletTTLMgr::generate_ttl_dag(ObTTLTaskInfo& task_info, ObTTLTaskParam& para) +{ + int ret = OB_SUCCESS; + ObTableTTLDag *dag = nullptr; + ObTableTTLDeleteTask *delete_task = nullptr; + + ObTenantDagScheduler *dag_scheduler = nullptr; + if (OB_ISNULL(dag_scheduler = MTL(ObTenantDagScheduler *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag scheduler must not be NULL", K(ret)); + } else if (OB_FAIL(dag_scheduler->alloc_dag(dag))) { + LOG_WARN("fail to alloc dag", KR(ret)); + } else if (OB_ISNULL(dag)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, dag is null", KR(ret), KP(dag)); + } else if (OB_FAIL(dag->init(para, task_info))) { + LOG_WARN("fail to init ttl dag", KR(ret)); + } else if (OB_FAIL(dag->alloc_task(delete_task))) { + LOG_WARN("fail to alloc ttl delete task", KR(ret)); + } else if (OB_ISNULL(delete_task)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, prepare task is null", KR(ret), KP(delete_task)); + } else if (OB_FAIL(delete_task->init(this, para, task_info))) { + LOG_WARN("fail to init ttl delete task", KR(ret)); + } else if (OB_FAIL(dag->add_task(*delete_task))) { + LOG_WARN("fail to add ttl delete task to dag", KR(ret)); + } else if (OB_FAIL(dag_scheduler->add_dag(dag))) { + if (OB_EAGAIN == ret) { + LOG_DEBUG("ttl dag already exists, no need to schedule once again", KR(ret)); + } else if (OB_SIZE_OVERFLOW == ret) { + LOG_DEBUG("dag is full", KR(ret)); + } else { + LOG_WARN("fail to add dag to queue", KR(ret)); + } + } else { + FLOG_INFO("build ttl dag success", KR(ret), K(para), K(task_info)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(dag_scheduler) && OB_NOT_NULL(dag)) { + dag_scheduler->free_dag(*dag); + } + LOG_INFO("finish to build table api ttl dag", KR(ret), K(para), K(task_info)); + return ret; +} + +int ObTenantTabletTTLMgr::handle_all_tablet_event(common::ObSArray& tablets) +{ + int ret = OB_SUCCESS; + ObTTLTaskCtx* ctx = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else { + common::ObSpinLockGuard guard(lock_); + for (tablet_task_iter iter = local_tenant_task_.tablet_task_map_.begin(); + iter != local_tenant_task_.tablet_task_map_.end(); ++iter) { + ctx = iter->second; + if (OB_ISNULL(ctx)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("fatal err, ttl ctx in map is null", KR(ret)); + } else if (OB_FAIL(handle_one_tablet_event(ctx))) { + LOG_WARN("fail to handle one tablet event", KR(ret)); + } else if (ctx->is_dirty_ && OB_FAIL(tablets.push_back(ctx->task_info_.tablet_id_))) { + LOG_WARN("fail to push back ttl pk", KR(ret)); + } + } + } + + LOG_DEBUG("finish handle single tenant event", KR(ret), K_(tenant_id)); + return ret; +} + +int ObTenantTabletTTLMgr::handle_one_tablet_event(ObTTLTaskCtx* ctx) +{ + int ret = OB_SUCCESS; + bool try_schedule = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ttl task ctx is null", KR(ret), K_(tenant_id), K(ctx)); + } else if (ctx->task_status_ != local_tenant_task_.state_) { + if (OB_TTL_TASK_RUNNING == local_tenant_task_.state_) { + if (OB_TTL_TASK_PENDING == ctx->task_status_) { + try_schedule = true; + } else if (OB_TTL_TASK_PREPARE == ctx->task_status_ || + OB_TTL_TASK_FINISH == ctx->task_status_ || + OB_TTL_TASK_CANCEL == ctx->task_status_) { + // do nothing + } else { + LOG_WARN("no expected task status", K(local_tenant_task_), KPC(ctx)); + } + } else if (OB_TTL_TASK_PENDING == local_tenant_task_.state_) { + if (OB_TTL_TASK_RUNNING == ctx->task_status_ || + OB_TTL_TASK_PREPARE == ctx->task_status_) { + ctx->task_status_ = local_tenant_task_.state_; + mark_ttl_ctx_dirty(local_tenant_task_, *ctx); + } else if (OB_TTL_TASK_FINISH == local_tenant_task_.state_){ + // do nothing, no need schedule finish task again + } else { + LOG_WARN("no expected task status", K(local_tenant_task_), KPC(ctx)); + } + } else if (OB_TTL_TASK_CANCEL == local_tenant_task_.state_) { + if (OB_TTL_TASK_PREPARE == ctx->task_status_ || + OB_TTL_TASK_RUNNING == ctx->task_status_ || + OB_TTL_TASK_PENDING == ctx->task_status_ || + OB_TTL_TASK_FINISH == ctx->task_status_) { + ctx->task_status_ = local_tenant_task_.state_; + mark_ttl_ctx_dirty(local_tenant_task_, *ctx); + } else { + LOG_WARN("no expected task status", K(local_tenant_task_), KPC(ctx)); + } + } else if (OB_TTL_TASK_MOVING == local_tenant_task_.state_) { + if (OB_TTL_TASK_PREPARE == ctx->task_status_) { + ctx->task_status_ = OB_TTL_TASK_FINISH; // will refresh real status from task table + mark_ttl_ctx_dirty(local_tenant_task_, *ctx); + } else if (OB_TTL_TASK_FINISH == ctx->task_status_ || + OB_TTL_TASK_CANCEL == ctx->task_status_) { + // do nothing, normal partition task + } else { + LOG_WARN("no expected task status", K(local_tenant_task_), KPC(ctx)); + } + } else { + LOG_WARN("invalid ttl tenant task state", K(local_tenant_task_.state_)); + } + + if (OB_SUCC(ret) && try_schedule && OB_FAIL(try_schedule_task(ctx))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("fail to try schedule dag task", KR(ret)); + } else { + ret = OB_SUCCESS; + } + } + LOG_DEBUG("handle one partition event", KR(ret), K(ctx->task_status_), K(local_tenant_task_.state_)); + } + return ret; +} + +int ObTenantTabletTTLMgr::check_tenant_memory() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet ttl mgr not init", KR(ret)); + } else if (!is_paused_) { + common::ObSpinLockGuard guard(lock_); + bool last_ttl_continue = local_tenant_task_.ttl_continue_; + int64_t total_memstore_used = 0; + int64_t minor_freeze_trigger = 0; + if (!common::ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + local_tenant_task_.ttl_continue_ = false; + } else { + int64_t active_memstore_used = 0; + int64_t memstore_limit = 0; + int64_t freeze_cnt = 0; + if (OB_FAIL(MTL(storage::ObTenantFreezer *)->get_tenant_memstore_cond( + active_memstore_used, + total_memstore_used, + minor_freeze_trigger, + memstore_limit, + freeze_cnt)) ) { + LOG_WARN("fail to get tenant memstore info for tenant ", KR(ret), K_(tenant_id)); + } else if (total_memstore_used > minor_freeze_trigger) { + local_tenant_task_.ttl_continue_ = false; + } else { + local_tenant_task_.ttl_continue_ = true; + } + } + if (last_ttl_continue != local_tenant_task_.ttl_continue_) { + FLOG_INFO("ttl continue value changed", K_(tenant_id), "ttl_continue", + local_tenant_task_.ttl_continue_, K(total_memstore_used), K(minor_freeze_trigger)); + } + } + + return ret; +} + +int ObTenantTabletTTLMgr::get_ttl_para_from_schema(const schema::ObTableSchema *table_schema, + ObTTLTaskParam ¶m) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("schema is null", KR(ret)); + } else if (!table_schema->get_kv_attributes().empty()) { + if (OB_FAIL(ObHTableUtils::check_htable_schema(*table_schema))) { + LOG_WARN("fail to check htable schema", KR(ret), K(table_schema->get_table_name())); + } else if (OB_FAIL(ObTTLUtil::parse_kv_attributes(table_schema->get_kv_attributes(), param.max_version_, param.ttl_))) { + LOG_WARN("fail to parse kv attributes", KR(ret), K(table_schema->get_kv_attributes())); + } else { + param.is_htable_ = true; + LOG_DEBUG("success to find a hbase ttl partition", KR(ret), K(param)); + } + } else if (!table_schema->get_ttl_definition().empty()) { + LOG_DEBUG("success to find a table ttl partition", KR(ret), K(param)); + } + + if (OB_SUCC(ret)) { + param.tenant_id_ = table_schema->get_tenant_id(); + param.database_id_ = table_schema->get_database_id(); + param.user_id_ = table_schema->get_define_user_id(); + param.table_id_ = table_schema->get_table_id(); + } + return ret; +} + +int ObTenantTabletTTLMgr::try_schedule_prepare_task(ObTabletID& tablet_id) +{ + int ret = OB_SUCCESS; + common::ObSpinLockGuard guard(lock_); + ObTTLTaskCtx* ctx = get_one_tablet_ctx(tablet_id); + if (OB_ISNULL(ctx)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("ttl tenant info or ctx is null", KR(ret)); + } else if (ctx->task_status_ != OB_TTL_TASK_PREPARE) { + // do nothing + } else if (FALSE_IT(ctx->task_status_ = OB_TTL_TASK_PENDING)) { + } else if (OB_FAIL(try_schedule_task(ctx))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("fail to schedule task", KR(ret)); + } else { + ret = OB_SUCCESS; + } + } + LOG_DEBUG("try schedule prepare task", KR(ret)); + return ret; +} + +int ObTenantTabletTTLMgr::sync_sys_table(ObTabletID& tablet_id) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator(lib::ObLabel("TTLStatusRecord")); + ObTTLTaskCtx* ctx = nullptr; + { + common::ObSpinLockGuard guard(lock_); + ctx = get_one_tablet_ctx(tablet_id); + if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ctx is null", KR(ret)); + } else if (OB_UNLIKELY(!ctx->is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid partition task ctx", KR(ret), KPC(ctx)); + } else { + ctx->last_modify_time_ = ObTimeUtility::current_time(); + } + } + + // lock task ctx for update + common::ObSpinLockGuard ctx_guard(ctx->lock_); + if (OB_SUCC(ret) && OB_UNLIKELY(ctx->need_refresh_)) { + switch (ctx->task_status_) { + case OB_TTL_TASK_FINISH: // tenant_info must be in moving status + case OB_TTL_TASK_PREPARE: { + if (OB_FAIL(refresh_tablet_task(*ctx, true /*refresh_status*/, true))) { + LOG_WARN("fail to refresh partition task from task table", KR(ret)); + } else { + if (ctx->task_info_.err_code_ == OB_NOT_MASTER || + (ctx->task_status_ != OB_TTL_TASK_FINISH && ctx->task_status_ != OB_TTL_TASK_CANCEL)) { + ctx->task_status_ = OB_TTL_TASK_PREPARE; + } + ctx->need_refresh_ = false; + ctx->task_info_.err_code_ = OB_SUCCESS; + } + break; + } + case OB_TTL_TASK_RUNNING: + case OB_TTL_TASK_PENDING: + case OB_TTL_TASK_CANCEL: { + if (OB_FAIL(refresh_tablet_task(*ctx, false /*refresh_status*/))) { + LOG_WARN("fail to refresh partition task from task table", KR(ret)); + } else { + ctx->need_refresh_ = false; + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected ttl task status", KR(ret)); + break; + } + } + } + + if (OB_SUCC(ret)) { + common::ObTTLStatus ttl_record; + switch (ctx->task_status_) { + case OB_TTL_TASK_PREPARE: { + ObMySQLTransaction trans; + ObTTLStatusFieldArray filters; + common::ObTTLStatusArray ttl_records; + ObTTLStatusFieldArray filter; + bool commit = false; + int tmp_ret = OB_SUCCESS; + bool is_exists = false; + if (OB_FAIL(construct_task_record_filter(ctx->task_info_.task_id_, + ctx->task_info_.table_id_, + ctx->task_info_.tablet_id_, + filters))) { + LOG_WARN("fail to construct task record filter", KR(ret)); + } else if (OB_FAIL(trans.start(get_sql_proxy(), gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail to start transation", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(ObTTLUtil::check_ttl_task_exists(tenant_id_, trans, ctx->task_info_.task_id_, + ctx->task_info_.table_id_, ctx->task_info_.tablet_id_, is_exists))) { + LOG_WARN("fail to check ttl task exist"); + } else if (!is_exists) { + if (OB_FAIL(construct_sys_table_record(ctx, ttl_record))) { + LOG_WARN("fail to construct sys table record", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::insert_ttl_task(tenant_id_, share::OB_ALL_KV_TTL_TASK_TNAME, + trans, ttl_record))) { + LOG_WARN("fail to insert ttl task", KR(ret)); + } + } + + if (trans.is_started()) { + bool commit = (OB_SUCCESS == ret); + int tmp_ret = ret; + if (OB_FAIL(trans.end(commit))) { + LOG_WARN("faile to end trans", "commit", commit, KR(ret)); + } + ret = tmp_ret == OB_SUCCESS ? ret : tmp_ret; + } + + // change prepare state to running/pending + if (OB_SUCC(ret) && OB_FAIL(try_schedule_prepare_task(tablet_id))) { + LOG_WARN("fail to schedule prepare task", KR(ret)); + } + break; + } + case OB_TTL_TASK_FINISH: + case OB_TTL_TASK_RUNNING: + case OB_TTL_TASK_PENDING: + case OB_TTL_TASK_CANCEL: { + ObMySQLTransaction trans; + ObTTLStatusFieldArray filters; + common::ObTTLStatusArray ttl_records; + ObTTLStatusFieldArray filter; + bool commit = false; + int tmp_ret = OB_SUCCESS; + bool is_exists = false; + if (OB_FAIL(construct_task_record_filter(ctx->task_info_.task_id_, + ctx->task_info_.table_id_, + ctx->task_info_.tablet_id_, + filters))) { + LOG_WARN("fail to construct task record filter", KR(ret)); + } else if (OB_FAIL(trans.start(get_sql_proxy(), gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail to start transation", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(ObTTLUtil::check_ttl_task_exists(tenant_id_, trans, ctx->task_info_.task_id_, + ctx->task_info_.table_id_, ctx->task_info_.tablet_id_, is_exists))) { + LOG_WARN("fail to check ttl task exist"); + } else if (!is_exists) { + if (OB_FAIL(construct_sys_table_record(ctx, ttl_record))) { + LOG_WARN("fail to construct sys table record", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::insert_ttl_task(tenant_id_, share::OB_ALL_KV_TTL_TASK_TNAME, + trans, ttl_record))) { + LOG_WARN("fail to insert ttl task", KR(ret)); + } + } else { + if (OB_FAIL(construct_sys_table_record(ctx, ttl_record))) { + LOG_WARN("fail to construct sys table record", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::update_ttl_task_all_fields(tenant_id_, + share::OB_ALL_KV_TTL_TASK_TNAME, + trans, ttl_record))) { + LOG_WARN("fail to update ttl task in sys table", KR(ret), K(ttl_record)); + } + } + + if (trans.is_started()) { + bool commit = (OB_SUCCESS == ret); + int tmp_ret = ret; + if (OB_FAIL(trans.end(commit))) { + LOG_WARN("faile to end trans", "commit", commit, KR(ret)); + } + ret = tmp_ret == OB_SUCCESS ? ret : tmp_ret; + } + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected ttl task status", KR(ret)); + break; + } + } + } + + if (OB_SUCC(ret)) { + //mark ctx dirty false + if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ctx is null", KR(ret)); + } else { + ctx->is_dirty_ = false; + LOG_INFO("finish mark ctx dirty false", KR(ret), K(tablet_id), KPC(ctx)); + } + } + return ret; +} + +int ObTenantTabletTTLMgr::construct_sys_table_record(ObTTLTaskCtx* ctx, common::ObTTLStatus& ttl_record) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invaild null ttl task ctx", KR(ret)); + } else if (OB_UNLIKELY(!ctx->is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KPC(ctx)); + } else { + ttl_record.tenant_id_ = tenant_id_; + ttl_record.table_id_ = ctx->task_info_.table_id_; + ttl_record.tablet_id_ = ctx->task_info_.tablet_id_.id(); + ttl_record.task_id_ = ctx->task_info_.task_id_; + + ttl_record.task_start_time_ = ctx->task_start_time_; + ttl_record.task_update_time_ = ctx->last_modify_time_; + ttl_record.trigger_type_ = static_cast(ctx->task_info_.is_user_trigger_); + ttl_record.status_ = static_cast(ctx->task_status_); + + ttl_record.ttl_del_cnt_ = ctx->task_info_.ttl_del_cnt_; + ttl_record.max_version_del_cnt_ = ctx->task_info_.max_version_del_cnt_; + ttl_record.scan_cnt_ = ctx->task_info_.scan_cnt_; + ttl_record.row_key_ = ctx->task_info_.row_key_; // shallow copy + ttl_record.ret_code_ = common::ob_error_name(ctx->task_info_.err_code_); + } + return ret; +} + +int ObTenantTabletTTLMgr::construct_task_record_filter(const uint64_t& task_id, + const uint64_t& table_id, + ObTabletID& tablet_id, + ObTTLStatusFieldArray& filter) +{ + int ret = OB_SUCCESS; + ObTTLStatusField task_id_field; + task_id_field.field_name_ = ObString("task_id"); + task_id_field.type_ = ObTTLStatusField::UINT_TYPE; + task_id_field.data_.uint_ = task_id ; + + ObTTLStatusField table_id_field; + table_id_field.field_name_ = ObString("table_id"); + table_id_field.type_ = ObTTLStatusField::UINT_TYPE; + table_id_field.data_.uint_ = table_id; + + ObTTLStatusField partition_id_field; + partition_id_field.field_name_ = ObString("tablet_id"); + partition_id_field.type_ = ObTTLStatusField::UINT_TYPE; + partition_id_field.data_.uint_ = tablet_id.id(); + + if (OB_FAIL(filter.push_back(task_id_field))) { + LOG_WARN("failt to push back", KR(ret)); + } else if (OB_FAIL(filter.push_back(table_id_field))) { + LOG_WARN("failt to push back", KR(ret)); + } else if (OB_FAIL(filter.push_back(partition_id_field))) { + LOG_WARN("failt to push back", KR(ret)); + } + return ret; +} + +int ObTenantTabletTTLMgr::from_ttl_record(ObTabletID& tablet_id, common::ObTTLStatus& record, bool with_status /*true*/, bool with_err_code /*true*/) +{ + int ret = OB_SUCCESS; + common::ObSpinLockGuard guard(lock_); + ObTTLTaskCtx* ctx = get_one_tablet_ctx(tablet_id); + if (OB_ISNULL(ctx)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("unexpected null value", KR(ret), KP(ctx)); + } else if (tenant_id_ != record.tenant_id_ || + tablet_id.id() != record.tablet_id_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("record do not match", KR(ret), K(record)); + } else { + ctx->task_info_.tablet_id_ = tablet_id; + ctx->task_info_.task_id_ = record.task_id_; + ctx->task_start_time_ = record.task_start_time_; + ctx->last_modify_time_ = record.task_update_time_; + ctx->task_info_.is_user_trigger_ = record.trigger_type_ == TRIGGER_TYPE::USER_TRIGGER; + ctx->task_info_.ttl_del_cnt_ = record.ttl_del_cnt_; + ctx->task_info_.max_version_del_cnt_ = record.max_version_del_cnt_; + ctx->task_info_.scan_cnt_ = record.scan_cnt_; + if (with_err_code) { + if (record.ret_code_.compare("OB_SUCCESS") == 0) { + ctx->task_info_.err_code_ = OB_SUCCESS; + } else if (record.ret_code_.compare("OB_NOT_MASTER") == 0) { + ctx->task_info_.err_code_ = OB_NOT_MASTER; + } else { + ctx->task_info_.err_code_ = OB_INVALID_ERROR; + } + } + if (with_status) { + ctx->task_status_ = static_cast(record.status_); + } + if (!record.row_key_.empty()) { + char *rowkey_buf = static_cast(local_tenant_task_.allocator_.alloc(record.row_key_.length())); + if (OB_ISNULL(rowkey_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", KR(ret)); + } else { + MEMCPY(rowkey_buf, record.row_key_.ptr(), record.row_key_.length()); + ctx->task_info_.row_key_.assign(rowkey_buf, record.row_key_.length()); + } + } + } + LOG_DEBUG("finish from ttl record", KR(ret), K(tablet_id)); + return ret; +} + +bool ObTenantTabletTTLMgr::can_schedule_tenant(const ObTTLTenantInfo &tenant_info) +{ + return tenant_info.ttl_continue_ && tenant_info.state_ == OB_TTL_TASK_RUNNING; +} + +bool ObTenantTabletTTLMgr::can_schedule_task(const ObTTLTaskCtx &ttl_task) +{ + return ttl_task.task_status_ == OB_TTL_TASK_PENDING; +} + +int ObTenantTabletTTLMgr::try_schedule_remaining_tasks(const ObTTLTaskCtx *current_ctx) +{ + int ret = OB_SUCCESS; + if (can_schedule_tenant(local_tenant_task_)) { + ObTTLTaskCtx* ctx = nullptr; + for (tablet_task_iter iter = local_tenant_task_.tablet_task_map_.begin(); + iter != local_tenant_task_.tablet_task_map_.end() + && OB_SUCC(ret); ++iter) { + ctx = iter->second; + if (OB_ISNULL(ctx)) { + ret = OB_ERR_NULL_VALUE; + LOG_ERROR("fatal err, ttl ctx in map is null", KR(ret), K(local_tenant_task_.tenant_id_)); + } else if (current_ctx == ctx) { + // do nothing + } else if (can_schedule_task(*ctx)) { + if (OB_FAIL(try_schedule_task(ctx))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("fail to schedule task", KR(ret)); + } + } + } + } + if (OB_SIZE_OVERFLOW == ret) { + ret = OB_SUCCESS; + } + } + return ret; +} + +// try schedule partition task, reutrn OB_SIZE_OVERFLOW if dag scheduler is full +int ObTenantTabletTTLMgr::try_schedule_task(ObTTLTaskCtx* ctx) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K_(tenant_id)); + } else if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ttl task ctx is null", KR(ret)); + } else if (can_schedule_tenant(local_tenant_task_) && can_schedule_task(*ctx)) { + if (OB_FAIL(generate_ttl_dag(ctx->task_info_, ctx->ttl_para_))) { + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; + } else if (OB_SIZE_OVERFLOW == ret) { + // do noting + } else { + LOG_WARN("fail to generate dag task", KR(ret)); + } + } else { + if (ctx->task_start_time_ == OB_INVALID_ID) { + ctx->task_start_time_ = ObTimeUtility::current_time(); + } + ctx->task_status_ = OB_TTL_TASK_RUNNING; + mark_ttl_ctx_dirty(local_tenant_task_, *ctx); + } + } else { + LOG_DEBUG("status when try schedule task", K(local_tenant_task_.ttl_continue_), K(local_tenant_task_.state_), K(ctx->task_status_)); + } + return ret; +} + +void ObTenantTabletTTLMgr::mark_ttl_ctx_dirty(ObTTLTenantInfo& tenant_info, ObTTLTaskCtx& ctx) +{ + ctx.is_dirty_ = true; + local_tenant_task_.is_dirty_ = true; +} + +int ObTenantTabletTTLMgr::refresh_tablet_task(ObTTLTaskCtx &ttl_task, bool refresh_status, bool refresh_retcode /*false*/) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + ObTTLStatusFieldArray filters; + common::ObTTLStatusArray ttl_records; + ObTTLStatusFieldArray filter; + ObTabletID tablet_id = ttl_task.task_info_.tablet_id_; + if (!ttl_task.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ttl_task)); + } else if (OB_FAIL(construct_task_record_filter(ttl_task.task_info_.task_id_, + ttl_task.task_info_.table_id_, + ttl_task.task_info_.tablet_id_, + filters))) { + LOG_WARN("fail to construct task record filter", KR(ret), K(ttl_task)); + } else if (OB_FAIL(trans.start(get_sql_proxy(), gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail to start transation", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(ObTTLUtil::read_ttl_tasks(tenant_id_, share::OB_ALL_KV_TTL_TASK_TNAME, + trans, filters, ttl_records, true, &local_tenant_task_.allocator_))) { + LOG_WARN("fail to get ttl tasks", KR(ret), K_(tenant_id), K(filters)); + } else { + if (ttl_records.empty()) { + // do nothing + } else { + if (ttl_records.count() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect ttl records count", KR(ret), K(ttl_records.count())); + } else if (OB_FAIL(from_ttl_record(ttl_task.task_info_.tablet_id_, ttl_records.at(0), refresh_status, refresh_retcode))) { + LOG_WARN("fail to convert from ttl record", KR(ret), K(refresh_status)); + } + } + } + if (trans.is_started()) { + bool commit = (OB_SUCCESS == ret); + int tmp_ret = ret; + if (OB_FAIL(trans.end(commit))) { + LOG_WARN("faile to end trans", "commit", commit, KR(ret)); + } + ret = tmp_ret == OB_SUCCESS ? ret : tmp_ret; + } + + return ret; +} + +// 1. scan and get the latest tenant ttl task +// 2. check the status and change local tenant info +int ObTenantTabletTTLMgr::reload_tenant_task() +{ + int ret = OB_SUCCESS; + ObTTLStatus tenant_task; + common::ObSpinLockGuard guard(lock_); + ObTTLTaskStatus expected_state; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (is_paused_) { + // do nothing, not leader + } else if (OB_FAIL(ObTTLUtil::read_tenant_ttl_task(tenant_id_, *sql_proxy_, tenant_task))) { + if (OB_ITER_END == ret) { + // do nothing + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to read tenant ttl task", KR(ret), K_(tenant_id)); + } + } else if (!local_tenant_task_.is_finished_ && local_tenant_task_.task_id_ != tenant_task.task_id_) { + FLOG_INFO("tenant task is finished, but local tenant task is not, maybe schema changed", + KR(ret), K_(local_tenant_task), K(tenant_task.task_id_)); + local_tenant_task_.reuse(); + } else if (OB_RS_TTL_TASK_MOVE == static_cast(tenant_task.status_)) { + // do nothing, wait tenant ttl manager finish move + } else if (OB_FAIL(transform_tenant_state(static_cast(tenant_task.status_), expected_state))) { + LOG_WARN("fail to transform ttl tenant task status", KR(ret), K(tenant_task.status_)); + } else if (OB_FAIL(check_cmd_state_valid(local_tenant_task_.state_, expected_state))) { + LOG_WARN("ttl cmd state machine is wrong", KR(ret), K_(tenant_id), K(tenant_task), K(expected_state)); + } else { + if (local_tenant_task_.is_finished_ && local_tenant_task_.task_id_ != tenant_task.task_id_) { + // new ttl request + local_tenant_task_.task_id_ = tenant_task.task_id_; + local_tenant_task_.is_usr_trigger_ = (tenant_task.trigger_type_ == USER_TRIGGER); + local_tenant_task_.state_ = expected_state; + local_tenant_task_.need_check_ = true; + local_tenant_task_.is_dirty_ = true; + local_tenant_task_.is_finished_ = false; + FLOG_INFO("new ttl task", KR(ret), K_(tenant_id), K_(local_tenant_task)); + } else if (local_tenant_task_.task_id_ != tenant_task.task_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task id is mismatch", KR(ret), K(local_tenant_task_.task_id_), K(tenant_task.task_id_)); + } else if (!local_tenant_task_.is_finished_ && local_tenant_task_.state_ != expected_state) { + FLOG_INFO("old ttl task changed", KR(ret), K_(tenant_id), K_(local_tenant_task), K(tenant_task)); + // current tenant task status changed + local_tenant_task_.state_ = expected_state; + local_tenant_task_.is_dirty_ = true; + } + } + return ret; +} + +int ObTenantTabletTTLMgr::check_schema_version() +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard schema_guard; + int64_t schema_version = 0; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret), K_(tenant_id)); + } else if (OB_FAIL(schema_guard.get_schema_version(tenant_id_, schema_version))) { + LOG_WARN("fail to get tenant schema version", K(ret), K_(tenant_id)); + } else if (!ObSchemaService::is_formal_version(schema_version)) { + ret = OB_EAGAIN; + LOG_INFO("is not a formal_schema_version", KR(ret), K(schema_version)); + } else if (local_schema_version_ == OB_INVALID_VERSION || local_schema_version_ < schema_version) { + FLOG_INFO("schema changed, mark tenant need check", KR(ret), K_(local_schema_version), K(schema_version)); + local_schema_version_ = schema_version; + mark_tenant_need_check(); + } + return ret; +} + +} // table +} // oceanbase diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h new file mode 100644 index 0000000000..5f794afa4d --- /dev/null +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h @@ -0,0 +1,282 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_TABLE_OB_TENANT_TABLET_TTL_MANAGER_H_ +#define OCEANBASE_OBSERVER_TABLE_OB_TENANT_TABLET_TTL_MANAGER_H_ + +#include "lib/task/ob_timer.h" +#include "share/table/ob_ttl_util.h" +#include "share/table/ob_table_ttl_common.h" +#include "share/tablet/ob_tablet_info.h" + +namespace oceanbase +{ +namespace table +{ + +struct ObTTLTaskCtx +{ +public : + ObTTLTaskCtx() : task_info_(), + task_status_(common::ObTTLTaskStatus::OB_TTL_TASK_INVALID), + ttl_para_(), + task_start_time_(OB_INVALID_ID), + last_modify_time_(OB_INVALID_ID), + failure_times_(0), + is_dirty_(false), + need_refresh_(true) {} + bool is_valid() + { + return task_info_.is_valid() && ttl_para_.is_valid(); + } + + TO_STRING_KV(K_(task_info), K_(task_status), K_(ttl_para), K_(task_start_time), + K_(last_modify_time), K_(failure_times), K_(is_dirty), K_(need_refresh)); + +public: + ObTTLTaskInfo task_info_; + common::ObTTLTaskStatus task_status_; + table::ObTTLTaskParam ttl_para_; + int64_t task_start_time_; + int64_t last_modify_time_; + int64_t failure_times_; + + int64_t rsp_time_; + bool is_invalid_; + bool is_dirty_; // should sync sys table for tasks + bool is_moved_; + bool need_refresh_; // should refresh task from task table + common::ObSpinLock lock_; // lock for update +}; + +class ObTenantTabletTTLMgr; +class OBTTLTimerPeriodicTask : public common::ObTimerTask { +public: + OBTTLTimerPeriodicTask(ObTenantTabletTTLMgr &tablet_ttl_mgr) + : tablet_ttl_mgr_(tablet_ttl_mgr) + {} + virtual ~OBTTLTimerPeriodicTask() {} + virtual void runTimerTask() override; +private: + ObTenantTabletTTLMgr &tablet_ttl_mgr_; +}; + +class ObTenantTabletTTLMgr : public logservice::ObIReplaySubHandler, + public logservice::ObICheckpointSubHandler, + public logservice::ObIRoleChangeSubHandler +{ +public: + friend ObTTLTaskCtx; + ObTenantTabletTTLMgr() + : tenant_id_(common::OB_INVALID_TENANT_ID), + schema_service_(NULL), + sql_proxy_(NULL), + allocator_(ObMemAttr(MTL_ID(), "TenantTTLMgr")), + is_inited_(false), + is_timer_start_(false), + periodic_delay_(TTL_PERIODIC_DELAY), + periodic_task_(*this), + ls_(nullptr), + tg_id_(0), + local_schema_version_(OB_INVALID_VERSION), + has_start_(false), + is_paused_(false) + { + } + + virtual ~ObTenantTabletTTLMgr() + { + } + + int init(storage::ObLS *ls); + + int init(const uint64_t tenant_id); + + int flush(share::SCN &rec_scn) + { + UNUSED(rec_scn); + return OB_SUCCESS; + } + share::SCN get_rec_scn() override { return share::SCN::max_scn(); } + + // for replay, do nothing + int replay(const void *buffer, + const int64_t buf_size, + const palf::LSN &lsn, + const share::SCN &scn) override + { + UNUSED(buffer); + UNUSED(buf_size); + UNUSED(lsn); + UNUSED(scn); + return OB_SUCCESS; + } + + // switch leader + int switch_to_leader(); + int resume_leader() { return switch_to_leader(); } + + int switch_to_follower_gracefully(); + void switch_to_follower_forcedly(); + void inner_switch_to_follower(); + + int start(); + void wait(); + void stop(); + void destroy(); + + int reload_tenant_task(); + int report_task_status(ObTTLTaskInfo& task_info, + table::ObTTLTaskParam& task_para, + bool& is_stop); + void on_schema_changed(uint64_t schema_changed_tenant_id); + + // timer handle function + virtual int check_and_generate_tablet_tasks(); + int check_and_handle_event(); + int check_tenant_memory(); + int check_inner_stat(); +private: + typedef common::hash::ObHashMap TabletTaskMap; + typedef TabletTaskMap::iterator tablet_task_iter; + + struct ObTTLTenantInfo + { + public: + ObTTLTenantInfo() : tablet_task_map_(), + allocator_(ObMemAttr(MTL_ID(), "TTLTenantInfo")), + tenant_id_(OB_INVALID_ID), + task_id_(OB_INVALID_ID), + is_usr_trigger_(false), + need_check_(false), + is_dirty_(false), + ttl_continue_(true), + cmd_type_(obrpc::ObTTLRequestArg::TTL_INVALID_TYPE), + rsp_time_(OB_INVALID_ID), + state_(common::ObTTLTaskStatus::OB_TTL_TASK_INVALID), + is_droped_(false), + is_finished_(true) + {} + ~ObTTLTenantInfo() + { + destory(); + } + void destory() + { + tablet_task_map_.destroy(); + allocator_.reset(); + } + void reuse() + { + tablet_task_map_.reuse(); + allocator_.reuse(); + is_usr_trigger_ = false; + need_check_ = false; + is_dirty_ = false; + ttl_continue_ = true; + state_ = common::ObTTLTaskStatus::OB_TTL_TASK_INVALID; + is_finished_ = true; + } + + TO_STRING_KV(K_(tenant_id), + K_(task_id), + K_(is_usr_trigger), + K_(need_check), + K_(is_dirty), + K_(ttl_continue), + K_(rsp_time), + K_(state), + K_(is_finished)); + + public: + TabletTaskMap tablet_task_map_; + common::ObArenaAllocator allocator_; + uint64_t tenant_id_; + int64_t task_id_; + bool is_usr_trigger_; + bool need_check_; /*need scan partition & check*/ + bool is_dirty_; /*need check the current ctx task*/ + bool ttl_continue_; + obrpc::ObTTLRequestArg::TTLRequestType cmd_type_; // deprecated @dazhi + int64_t rsp_time_; // OB_INVALID_ID means no need response + common::ObTTLTaskStatus state_; + bool is_droped_; // tenant is droped + bool is_finished_; // all delete task is finished (or canceled) + }; + + int alloc_tenant_info(uint64_t tenant_id); + ObTTLTaskCtx* get_one_tablet_ctx(const common::ObTabletID& tablet_id); + + // /*inner function*/ + int deep_copy_task(ObTTLTaskCtx* ctx, table::ObTTLTaskInfo& task_info, const table::ObTTLTaskParam &task_param); + int handle_all_tablet_event(common::ObSArray& tablets); + int handle_one_tablet_event(ObTTLTaskCtx* ctx); + int generate_batch_tablet_task(ObIArray& tablet_pairs, + hash::ObHashMap ¶m_map); + int generate_one_tablet_task(table::ObTTLTaskInfo& task_info, const table::ObTTLTaskParam& para); + int get_ttl_para_from_schema(const schema::ObTableSchema *table_schema, table::ObTTLTaskParam& param); + void mark_tenant_need_check(); + virtual int generate_ttl_dag(table::ObTTLTaskInfo& task_info, table::ObTTLTaskParam& para); + static int construct_task_record_filter(const uint64_t& task_id, + const uint64_t& table_id, + ObTabletID& tablet_id, + ObTTLStatusFieldArray& filter); + common::ObMySQLProxy *get_sql_proxy() { return sql_proxy_; } + int sync_sys_table(common::ObTabletID& tablet_id); + int construct_sys_table_record(ObTTLTaskCtx* ctx, common::ObTTLStatus& ttl_record); + int try_schedule_task(ObTTLTaskCtx* ctx); + int try_schedule_remaining_tasks(const ObTTLTaskCtx *current_ctx); + bool can_schedule_tenant(const ObTTLTenantInfo &tenant_info); + bool can_schedule_task(const ObTTLTaskCtx &ttl_task); + int check_cmd_state_valid(const common::ObTTLTaskStatus current_state, + const common::ObTTLTaskStatus incoming_state); + int copy_all_tenant_ctxs(common::ObSArray& ctx_array, uint64_t tenant_id); + int from_ttl_record(ObTabletID& tablet_id, common::ObTTLStatus& record, bool with_status = true, bool with_err_code = true); + void mark_ttl_ctx_dirty(ObTTLTenantInfo& tenant_info, ObTTLTaskCtx& ctx); + void check_ttl_tenant_state(); + int transform_tenant_state(const common::ObTTLTaskStatus& tenant_status, common::ObTTLTaskStatus& status); + int try_schedule_prepare_task(ObTabletID& tablet_id); + void mark_tenant_checked(); + int refresh_tablet_task(ObTTLTaskCtx &ttl_task, bool refresh_status, bool refresh_retcode = false); + int check_schema_version(); + void resume(); + void pause(); + +private: + static const int64_t DEFAULT_TTL_BUCKET_NUM = 100; + static const int64_t TTL_PERIODIC_DELAY = 5*1000*1000; //5s + static const int64_t TBALE_GENERATE_BATCH_SIZE = 200; + static const int64_t DEFAULT_TABLE_ARRAY_SIZE = 200; + static const int64_t DEFAULT_TABLET_PAIR_SIZE = 1024; + static const int64_t DEFAULT_PARAM_BUCKET_SIZE = 200; + uint64_t tenant_id_; + ObTTLTenantInfo local_tenant_task_; + share::schema::ObMultiVersionSchemaService *schema_service_; + common::ObMySQLProxy *sql_proxy_; + common::ObArenaAllocator allocator_; + bool is_inited_; + bool is_timer_start_; + int64_t periodic_delay_; + OBTTLTimerPeriodicTask periodic_task_; + common::ObSpinLock lock_; + storage::ObLS *ls_; + int tg_id_; + ObArray tablet_table_pairs_; + int64_t local_schema_version_; + bool has_start_; + bool is_paused_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_TABLE_OB_TENANT_TABLET_TTL_MANAGER_H_ */ diff --git a/src/observer/table/ttl/ob_tenant_ttl_manager.cpp b/src/observer/table/ttl/ob_tenant_ttl_manager.cpp new file mode 100644 index 0000000000..2c3ef5c78e --- /dev/null +++ b/src/observer/table/ttl/ob_tenant_ttl_manager.cpp @@ -0,0 +1,762 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER + +#include "observer/table/ttl/ob_tenant_ttl_manager.h" +#include "share/ob_max_id_fetcher.h" +#include "share/table/ob_ttl_util.h" +#include "lib/oblog/ob_log_module.h" + +using namespace oceanbase::share; +using namespace oceanbase::common; + +namespace oceanbase +{ + +namespace table +{ + +void ObClearTTLHistoryTask::runTimerTask() +{ + ObCurTraceId::init(GCONF.self_addr_); + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ob clear ttl history task is not init", KR(ret)); + } else if (ObTTLUtil::check_can_do_work()) { + int ret = OB_SUCCESS; + const int64_t now = ObTimeUtility::current_time(); + ObSqlString sql; + if (tenant_id_ == OB_SYS_TENANT_ID) { + } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + // do nothinig + } else { + sql.reuse(); + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); + if (!tenant_config.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail get tenant_config", KR(ret), K(tenant_id_)); + } else { + int64_t delete_timestamp = now - tenant_config->kv_ttl_history_recycle_interval; + int64_t affect_rows = 0; + + if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE task_update_time < %ld LIMIT %ld ", + share::OB_ALL_KV_TTL_TASK_HISTORY_TNAME, + delete_timestamp, + OB_KV_TTL_GC_COUNT_PER_TASK))) { + LOG_WARN("fail to assign fmt sql string", KR(ret)); + } else if (OB_FAIL(sql_proxy_->write(gen_meta_tenant_id(tenant_id_), sql.ptr(), affect_rows))) { + LOG_WARN("fail to execute sql", KR(ret), K(sql)); + } else { + LOG_DEBUG("success to execute sql", KR(ret), K(sql)); + } + } + } + } +} + +int ObClearTTLHistoryTask::init(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ttl history task init twice", KR(ret)); + } else if (tenant_id == OB_INVALID_TENANT_ID) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", KR(ret), K(tenant_id)); + } else { + sql_proxy_ = &sql_proxy; + tenant_id_ = tenant_id; + is_inited_ = true; + } + return ret; +} + +int ObTTLTaskScheduler::init(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ttl task scheduler init twice", KR(ret)); + } else { + tablet_table_pairs_.set_attr(ObMemAttr(tenant_id, "TTLTabletPairs")); + sql_proxy_ = &sql_proxy; + tenant_id_ = tenant_id; + is_inited_ = true; + } + return ret; +} + +int ObTTLTaskScheduler::reload_tenant_task() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + // do nothing + } else if (need_reload_) { + lib::ObMutexGuard guard(mutex_); + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObTTLStatusField table_id_field; + table_id_field.field_name_ = ObString("table_id"); + table_id_field.type_ = ObTTLStatusField::UINT_TYPE; + table_id_field.data_.uint_ = OB_INVALID_ID; + + ObTTLStatusField partition_id_field; + partition_id_field.field_name_ = ObString("tablet_id"); + partition_id_field.type_ = ObTTLStatusField::UINT_TYPE; + partition_id_field.data_.uint_ = OB_INVALID_ID; + + ObTTLStatusField tenant_id_field; + tenant_id_field.field_name_ = ObString("tenant_id"); + tenant_id_field.type_ = ObTTLStatusField::UINT_TYPE; + tenant_id_field.data_.uint_ = tenant_id_; + + ObTTLStatusFieldArray filters; + ObTTLStatusArray ttl_task_arr; + if (OB_FAIL(filters.push_back(table_id_field)) || + OB_FAIL(filters.push_back(partition_id_field)) || + OB_FAIL(filters.push_back(tenant_id_field))) { + LOG_WARN("fail to push back field into array", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::read_ttl_tasks(tenant_id_, + share::OB_ALL_KV_TTL_TASK_TNAME, + *sql_proxy_, filters, ttl_task_arr))) { + LOG_WARN("fail to read ttl tasks status", KR(ret)); + } else if (ttl_task_arr.empty()) { + // do nothing + } else if (ttl_task_arr.size() == 1) { + ObTTLStatus &task = ttl_task_arr.at(0); + tenant_task_.ttl_status_ = task; + tenant_task_.is_finished_ = false; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected tenant ttl task count", KR(ret)); + } + } + + if (OB_SUCC(ret)) { + set_need_reload(false); + FLOG_INFO("reload tenant task", K_(tenant_task)); + } + } + return ret; +} + +ObTTLTaskStatus ObTTLTaskScheduler::next_status(int64_t curr) +{ + ObTTLTaskStatus next = ObTTLTaskStatus::OB_TTL_TASK_INVALID; + ObTTLTaskStatus curr_status = EVAL_TASK_PURE_STATUS(curr); + if (curr_status == ObTTLTaskStatus::OB_RS_TTL_TASK_CREATE || + curr_status == ObTTLTaskStatus::OB_RS_TTL_TASK_CANCEL) { + next = ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE; + } else if (curr_status == ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE || + curr_status == ObTTLTaskStatus::OB_RS_TTL_TASK_SUSPEND) { + next = curr_status; + } + return next; +} + +int ObTTLTaskScheduler::calc_next_task_state(ObTTLTaskType user_cmd_type, + ObTTLTaskStatus curr_state, + ObTTLTaskStatus &next_state) +{ + int ret = OB_SUCCESS; + if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("fail to modify ttl tasks status, the state is not mismatch, moving now", + KR(ret), K(curr_state), K(user_cmd_type)); + LOG_USER_WARN(OB_NOT_SUPPORTED, "Change the current TTL task state(Move) to the destination state"); + } else if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_SUSPEND && + user_cmd_type == ObTTLTaskType::OB_TTL_RESUME) { + next_state = ObTTLTaskStatus::OB_RS_TTL_TASK_CREATE; + } else if (curr_state == ObTTLTaskStatus::OB_RS_TTL_TASK_CREATE && + user_cmd_type == ObTTLTaskType::OB_TTL_SUSPEND) { + next_state = ObTTLTaskStatus::OB_RS_TTL_TASK_SUSPEND; + } else if (curr_state != ObTTLTaskStatus::OB_RS_TTL_TASK_CANCEL && + user_cmd_type == ObTTLTaskType::OB_TTL_CANCEL) { + next_state = ObTTLTaskStatus::OB_RS_TTL_TASK_CANCEL; + } else { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("fail to modify ttl tasks status, the state is not mismatch", + KR(ret), K(curr_state), K(user_cmd_type)); + LOG_USER_WARN(OB_NOT_SUPPORTED, "Change the current TTL task state to the destination state"); + } + return ret; +} + +int ObTTLTaskScheduler::add_ttl_task(ObTTLTaskType task_type) +{ + int ret = OB_SUCCESS; + TRIGGER_TYPE trigger_type = TRIGGER_TYPE::USER_TRIGGER; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("cann't process ttl task, maybe tenant is restoring", K_(tenant_id), KR(ret)); + } else if (OB_FAIL(reload_tenant_task())) { + LOG_WARN("fail to reload tenant task", KR(ret)); + } else { + lib::ObMutexGuard guard(mutex_); + ObTTLTaskStatus next_state = OB_TTL_TASK_INVALID; + ObTTLTaskStatus curr_state = EVAL_TASK_PURE_STATUS(tenant_task_.ttl_status_.status_); + LOG_INFO("add ttl task old task", K(tenant_task_.ttl_status_), K(tenant_task_.is_finished_)); + if (!tenant_task_.is_finished_) { + if (OB_FAIL(calc_next_task_state(task_type, curr_state, next_state))) { + LOG_WARN("fail to apply user cmd on current ttl tasks", KR(ret)); + } else { + LOG_INFO("user task results:", K(task_type), K(curr_state), K(next_state)); + if (next_state == curr_state) { + // duplicate request, do nothing + } else if (next_state != curr_state) { + // can transform to next state + if (OB_FAIL(update_task_status(tenant_task_.ttl_status_.task_id_, + static_cast(next_state), *sql_proxy_))) { + LOG_WARN("fail to update ttl tasks", KR(ret)); + } else { + tenant_task_.ttl_status_.status_ = static_cast(next_state); + } + } + } + } else { + // not task or task finished already, only accept trigger command + if (task_type != ObTTLTaskType::OB_TTL_TRIGGER) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("not ttl task currently, only trigger command is supported", KR(ret), K(task_type)); + LOG_USER_WARN(OB_NOT_SUPPORTED, "trigger TTL task when TTL task is executing"); + } else if (OB_FAIL(add_ttl_task_internal(TRIGGER_TYPE::USER_TRIGGER))) { + LOG_WARN("fail to add ttl task", KR(ret), K_(tenant_id)); + } + } + } + + return ret; +} + + +int ObTTLTaskScheduler::add_ttl_task_internal(TRIGGER_TYPE trigger_type) +{ + int ret = OB_SUCCESS; + bool is_active_time = false; + bool enable_ttl = is_enable_ttl(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (!is_enable_ttl()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl is not enable currently", KR(ret)); + } else if (!tenant_task_.is_finished_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("there is ttl task executing already", KR(ret)); + } else if (OB_FAIL(fetch_ttl_task_id(tenant_id_, tenant_task_.ttl_status_.task_id_))) { + LOG_WARN("fail to fetch ttl task id", KR(ret)); + } else { + int64_t cur_time = ObTimeUtility::current_time(); + tenant_task_.ttl_status_.task_start_time_ = cur_time; + tenant_task_.ttl_status_.task_update_time_ = cur_time; + tenant_task_.ttl_status_.tenant_id_ = tenant_id_; + tenant_task_.ttl_status_.trigger_type_ = static_cast(trigger_type); + tenant_task_.ttl_status_.status_ = static_cast(ObTTLTaskStatus::OB_RS_TTL_TASK_CREATE); + tenant_task_.is_finished_ = false; + if (OB_FAIL(insert_tenant_task(tenant_task_.ttl_status_))) { + LOG_WARN("fail to insert ttl task into __all_ttl_task_status.", KR(ret)); + tenant_task_.reset(); + } + } + return ret; +} + +int ObTTLTaskScheduler::insert_tenant_task(ObTTLStatus& ttl_task) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::insert_ttl_task(ttl_task.tenant_id_, + share::OB_ALL_KV_TTL_TASK_TNAME, *sql_proxy_, ttl_task))) { + LOG_WARN("fail to insert tenant status.", KR(ret)); + } + LOG_INFO("finish insert tenant ttl task", KR(ret), K(ttl_task)); + return ret; +} + +int ObTTLTaskScheduler::update_task_status(uint64_t task_id, + int64_t status, + common::ObISQLClient& proxy) +{ + int ret = OB_SUCCESS; + ObTTLStatusKey key(tenant_id_, OB_INVALID_ID, OB_INVALID_ID, task_id); + ObTTLStatusFieldArray update_fields; + + ObTTLStatusField status_field; + status_field.field_name_ = ObString("status"); + status_field.type_ = ObTTLStatusField::INT_TYPE; + status_field.data_.int_ = status; + + ObTTLStatusField update_time_field; + update_time_field.field_name_ = ObString("task_update_time"); + update_time_field.type_ = ObTTLStatusField::INT_TYPE; + update_time_field.data_.int_ = static_cast(ObTimeUtility::current_time()); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (OB_FAIL(update_fields.push_back(status_field)) || + OB_FAIL(update_fields.push_back(update_time_field))) { + LOG_WARN("fail to push back update fields.", KR(ret)); + } else { + if (OB_FAIL(ObTTLUtil::update_ttl_task(tenant_id_, + share::OB_ALL_KV_TTL_TASK_TNAME, + proxy, + key, + update_fields))) { + LOG_WARN("fail to update ttl task status.", KR(ret), K(tenant_id_), K(task_id), K(status)); + } else { + LOG_DEBUG("success to update ttl tasks status", KR(ret), K(tenant_id_), K(task_id), K(status)); + } + } + + return ret; +} + +int ObTTLTaskScheduler::delete_task(const uint64_t tenant_id, const uint64_t task_id) +{ + int ret = OB_SUCCESS; + ObTTLStatusKey key(tenant_id_, OB_INVALID_ID, OB_INVALID_ID, task_id); + int64_t affected_rows = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::delete_ttl_task(tenant_id, + share::OB_ALL_KV_TTL_TASK_TNAME, + *sql_proxy_, key, affected_rows))) { + LOG_WARN("fail to delete ttl tasks status", KR(ret), K_(tenant_id), K(task_id)); + } else { + LOG_DEBUG("success to delete ttl tasks status", KR(ret), K_(tenant_id), K(task_id), K(affected_rows)); + } + + return ret; +} + +int ObTTLTaskScheduler::in_active_time(bool& is_active_time) +{ + int ret = OB_SUCCESS; + is_active_time = false; + ObTTLDutyDuration duration; + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); + + if (!tenant_config.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail get tenant_config", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(ObTTLUtil::parse(tenant_config->kv_ttl_duty_duration, duration))) { + LOG_WARN("fail parse ttl dury duration", KR(ret)); + } else if (ObTTLUtil::current_in_duration(duration)) { + is_active_time = true; + } + + return ret; +} + +bool ObTTLTaskScheduler::is_enable_ttl() +{ + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); + return tenant_config.is_valid() && tenant_config->enable_kv_ttl; +} + +int ObTTLTaskScheduler::try_add_periodic_task() +{ + int ret = OB_SUCCESS; + TRIGGER_TYPE trigger_type = TRIGGER_TYPE::PERIODIC_TRIGGER; + bool is_active_time = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("cann't process ttl task, maybe tenant is restoring", K_(tenant_id), KR(ret)); + } else if (OB_FAIL(in_active_time(is_active_time))) { + LOG_WARN("fail to check is in active time", KR(ret)); + } else if (is_active_time) { + if (!periodic_launched_) { + lib::ObMutexGuard guard(mutex_); + if (tenant_task_.is_finished_ && OB_FAIL(add_ttl_task_internal(TRIGGER_TYPE::PERIODIC_TRIGGER))) { + LOG_WARN("fail to add ttl task", KR(ret), K_(tenant_id)); + } else { + periodic_launched_ = false; + } + } + } else { + periodic_launched_ = false; + } + return ret; +} + +int ObTTLTaskScheduler::check_all_tablet_task() +{ + int ret = OB_SUCCESS; + bool need_move = true; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else if (tenant_task_.ttl_status_.status_ == OB_TTL_TASK_INVALID) { + // do nothing + } else if (!ObTTLUtil::check_can_process_tenant_tasks(tenant_id_)) { + // do nothing + } else if (OB_FAIL(check_task_need_move(need_move))) { + LOG_WARN("fail to check task need move", KR(ret), K_(tenant_id)); + } else if (need_move) { + { + lib::ObMutexGuard guard(mutex_); + if (ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE != tenant_task_.ttl_status_.status_) { + tenant_task_.ttl_status_.status_ = static_cast(ObTTLTaskStatus::OB_RS_TTL_TASK_MOVE); + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail start transaction", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(update_task_status(tenant_task_.ttl_status_.task_id_, OB_RS_TTL_TASK_MOVE, trans))) { + LOG_WARN("fail to update task status", KR(ret)); + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("fail to commit trans", KR(ret), K(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(move_all_task_to_history_table())) { + LOG_WARN("fail to move all tasks to history table", KR(ret), K_(tenant_id), K(tenant_task_.ttl_status_.table_id_)); + } else { + tenant_task_.reset(); + FLOG_INFO("tenant task is finished", K_(tenant_task)); + } + } + } + + return ret; +} + +int ObTTLTaskScheduler::check_one_tablet_task(common::ObISQLClient &sql_client, + const uint64_t table_id, + const ObTabletID tablet_id, + bool &is_finished) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + is_finished = false; + SMART_VAR(ObISQLClient::ReadResult, res) { + sqlclient::ObMySQLResult* result = nullptr; + if (OB_FAIL(sql.append_fmt( + "SELECT count(*) as cnt FROM %s WHERE tenant_id = %ld and task_id = %ld and " + " table_id = %ld and tablet_id = %ld and (status = %ld or status = %ld) ", + OB_ALL_KV_TTL_TASK_TNAME, + tenant_id_, + tenant_task_.ttl_status_.task_id_, + table_id, + tablet_id.id(), + static_cast(ObTTLTaskStatus::OB_TTL_TASK_FINISH), + static_cast(ObTTLTaskStatus::OB_TTL_TASK_CANCEL)))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, gen_meta_tenant_id(tenant_id_), sql.ptr()))) { + LOG_WARN("fail to execute sql", KR(ret), K_(tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get mysql result failed", KR(ret), K(sql)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("fail to get next result", KR(ret), K(sql)); + } else { + int64_t cnt = 0; + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "cnt", cnt, int64_t, + true/*skip_null_error*/, false/*skip_column_error*/, 0/*default value*/); + if (OB_SUCC(ret) and (1 == cnt)) { + is_finished = true; + } + } + } + return ret; +} + +int ObTTLTaskScheduler::fetch_ttl_task_id(uint64_t tenant_id, int64_t &new_task_id) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl tenant task mgr not init", KR(ret)); + } else { + uint64_t tmp_task_id = OB_INVALID_ID; + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + if (OB_FAIL(id_fetcher.fetch_new_max_id(tenant_id, + share::OB_MAX_USED_TTL_TASK_ID_TYPE, + tmp_task_id, 0))) { + LOG_WARN("fail to fetch new ttl task id", KR(ret), "id_type", share::OB_MAX_USED_TTL_TASK_ID_TYPE); + } else { + new_task_id = tmp_task_id; + } + } + return ret; +} + +void ObTTLTaskScheduler::reset_local_tenant_task() +{ + lib::ObMutexGuard guard(mutex_); + tenant_task_.reset(); +} + +int ObTenantTTLManager::init(const uint64_t tenant_id, ObMySQLProxy &sql_proxy) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("tenant ttl mgr init twice", KR(ret)); + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TenantTTLManager, tg_id_))) { + LOG_WARN("fail to init timer", KR(ret)); + } else if (OB_FAIL(task_scheduler_.init(tenant_id, sql_proxy))) { + LOG_WARN("fail to init task scheduler", K(tenant_id)); + } else if (OB_FAIL(clear_ttl_history_task_.init(tenant_id, sql_proxy))) { + LOG_WARN("fail to init clear history task", K(tenant_id)); + } else { + is_inited_ = true; + tenant_id_ = tenant_id; + LOG_INFO("tenant ttl mgr is inited", K_(tenant_id)); + } + return ret; +} + +int ObTenantTTLManager::start() +{ + int ret = OB_SUCCESS; + FLOG_INFO("tenant ttl manager begin to start", K_(tenant_id)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (FALSE_IT(task_scheduler_.set_need_reload(true))) { + } else if (OB_FAIL(TG_START(tg_id_))) { + LOG_WARN("init ttl scheduler fail", KR(ret)); + } else if (OB_FAIL(TG_SCHEDULE(tg_id_, task_scheduler_, SCHEDULE_PERIOD, true))) { + LOG_WARN("fail to schedule ttl task scheduler", KR(ret)); + } else if (OB_FAIL(TG_SCHEDULE(tg_id_, clear_ttl_history_task_, + ObClearTTLHistoryTask::OB_KV_TTL_GC_INTERVAL, true))) { + LOG_WARN("fail to start ttl clear history task", KR(ret)); + } + FLOG_INFO("tenant ttl manager finish to start", KR(ret), K_(tenant_id)); + + return ret; +} + +void ObTenantTTLManager::wait() +{ + FLOG_INFO("tenant ttl manager start to wait", K_(tenant_id)); + TG_WAIT(tg_id_); + FLOG_INFO("tenant ttl manager finish to wait", K_(tenant_id)); +} + +void ObTenantTTLManager::stop() +{ + FLOG_INFO("tenant ttl manager start to stop", K_(tenant_id)); + TG_STOP(tg_id_); + FLOG_INFO("tenant ttl manager finish to stop", K_(tenant_id)); +} + +void ObTenantTTLManager::destroy() +{ + FLOG_INFO("tenant ttl manager start to destroy", K_(tenant_id)); + TG_DESTROY(tg_id_); + FLOG_INFO("tenant ttl manager finish to destroy", K_(tenant_id)); +} + +void ObTTLTaskScheduler::runTimerTask() +{ + int ret = OB_SUCCESS; + ObCurTraceId::init(GCONF.self_addr_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ttl task mgr not init", KR(ret)); + } else if (OB_FAIL(reload_tenant_task())) { + LOG_WARN("fail to process tenant task", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(try_add_periodic_task())) { + LOG_WARN("fail to try add periodic task", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(check_all_tablet_task())) { + LOG_WARN("fail to check all tablet task", KR(ret), K_(tenant_id)); + } + uint64_t current = ObTimeUtility::current_time(); + LOG_DEBUG("runTimerTask", KR(ret), K_(tenant_id), K(current)); +} + +int ObTenantTTLManager::handle_user_ttl(const obrpc::ObTTLRequestArg& arg) +{ + int ret = OB_SUCCESS; + ObTTLTaskType user_ttl_req_type = static_cast(arg.cmd_code_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (tenant_id_ == OB_SYS_TENANT_ID) { + // do nothing + } else if (OB_FAIL(task_scheduler_.add_ttl_task(static_cast(arg.cmd_code_)))) { + LOG_WARN("fail to add ttl task", KR(ret), K_(tenant_id), K(user_ttl_req_type)); + } + + FLOG_INFO("finish handle user ttl cmd", KR(ret), K(arg), K_(tenant_id)); + return ret; +} + +int ObTTLTaskScheduler::move_all_task_to_history_table() +{ + int ret = OB_SUCCESS; + int64_t one_move_rows = TBALET_CHECK_BATCH_SIZE; + while (OB_SUCC(ret) && one_move_rows == TBALET_CHECK_BATCH_SIZE) { + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail start transaction", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(ObTTLUtil::move_task_to_history_table(tenant_id_, tenant_task_.ttl_status_.task_id_, + trans, TBALET_CHECK_BATCH_SIZE, one_move_rows))) { + LOG_WARN("fail to move task to history table", KR(ret), K_(tenant_id)); + } + + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("fail to commit trans", KR(ret), K(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } + + if (OB_SUCC(ret)) { + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail start transaction", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(update_task_status(tenant_task_.ttl_status_.task_id_, OB_TTL_TASK_FINISH, trans))) { + LOG_WARN("fail to update task status", KR(ret)); + } else if (OB_FAIL(ObTTLUtil::move_tenant_task_to_history_table(tenant_id_, tenant_task_.ttl_status_.task_id_, + trans))) { + LOG_WARN("fail to move tenant task to history table", KR(ret), K_(tenant_id)); + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("fail to commit trans", KR(ret), K(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } + + return ret; +} + +int ObTTLTaskScheduler::check_task_need_move(bool &need_move) +{ + int ret = OB_SUCCESS; + need_move = false; + if (OB_RS_TTL_TASK_MOVE == tenant_task_.ttl_status_.status_) { + need_move = true; + } else if (OB_FAIL(check_all_tabelt_finished(need_move))) { + LOG_WARN("fail to check all tablet task finished", KR(ret)); + } + return ret; +} + +int ObTTLTaskScheduler::check_all_tabelt_finished(bool &all_finished) +{ + int ret = OB_SUCCESS; + all_finished = true; + ObSEArray table_id_array; + if (OB_FAIL(ObTTLUtil::get_tenant_table_ids(tenant_id_, table_id_array))) { + LOG_WARN("fail to get tenant table ids", KR(ret), K_(tenant_id)); + } else if (!table_id_array.empty() && OB_FAIL(tablet_table_pairs_.reserve(DEFAULT_TABLET_PAIR_SIZE))) { + LOG_WARN("fail to reserve", KR(ret)); + } + + // 2. get all ttl table ids + int64_t start_idx = 0; + int64_t end_idx = 0; + while (OB_SUCC(ret) && start_idx < table_id_array.count() && all_finished) { + { + // temp schema guard to loop tablet table ids + ObSchemaGetterGuard schema_guard; + start_idx = end_idx; + end_idx = MIN(table_id_array.count(), start_idx + TBALE_CHECK_BATCH_SIZE); + bool is_ttl_table = false; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret), K_(tenant_id)); + } + for (int64_t idx = start_idx; OB_SUCC(ret) && idx < end_idx; ++idx) { + const int64_t table_id = table_id_array.at(idx); + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, table_id, table_schema))) { + LOG_WARN("failed to get simple schema", KR(ret), K(table_id)); + } else if (OB_FAIL(ObTTLUtil::check_is_ttl_table(*table_schema, is_ttl_table))) { + LOG_WARN("fail to check is ttl table", KR(ret)); + } else if (is_ttl_table) { + ObArray tablet_ids; + if (OB_FAIL(table_schema->get_tablet_ids(tablet_ids))) { + LOG_WARN("fail to get tablet ids", KR(ret), K(table_id)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { + ObTabletTablePair cur_pair; + ObTabletID tablet_id = tablet_ids[i]; + if (OB_FAIL(cur_pair.init(tablet_id, table_id))) { + LOG_WARN("fail to init tablet_ls_pair", KR(ret), K(i), K(tablet_id), K(table_id)); + } else if (OB_FAIL(tablet_table_pairs_.push_back(cur_pair))) { + LOG_WARN("fail to push back pair", KR(ret), K(cur_pair)); + } + } + } + } + } + } // end scope + if (OB_SUCC(ret) && OB_FAIL(check_tablet_table_finished(tablet_table_pairs_, all_finished))) { + LOG_WARN("fail to check tablet table finished", KR(ret)); + } + tablet_table_pairs_.reuse(); + } + tablet_table_pairs_.reset(); + + return ret; +} + +int ObTTLTaskScheduler::check_tablet_table_finished(ObIArray &pairs, bool &all_finished) +{ + int ret = OB_SUCCESS; + int64_t start_idx = 0; + int64_t end_idx = 0; + while (OB_SUCC(ret) && start_idx < pairs.count() && all_finished) { + start_idx = end_idx; + end_idx = MIN(pairs.count(), start_idx + TBALET_CHECK_BATCH_SIZE); + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) { + LOG_WARN("fail start transaction", KR(ret), K_(tenant_id)); + } else { + for (int64_t idx = start_idx; OB_SUCC(ret) && idx < end_idx && all_finished; ++idx) { + const int64_t table_id = pairs.at(idx).get_table_id(); + const ObTabletID tablet_id = pairs.at(idx).get_tablet_id(); + if (OB_FAIL(check_one_tablet_task(trans, table_id, tablet_id, all_finished))) { + LOG_WARN("fail to check tablet task", KR(ret), K(table_id), K(tablet_id)); + } + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("fail to commit trans", KR(ret), K(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } + return ret; +} + +} // end namespace table +} // end namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ttl/ob_tenant_ttl_manager.h b/src/observer/table/ttl/ob_tenant_ttl_manager.h new file mode 100644 index 0000000000..5106c8e5f5 --- /dev/null +++ b/src/observer/table/ttl/ob_tenant_ttl_manager.h @@ -0,0 +1,195 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_TABLE_OB_TENANT_TTL_MANAGER_H_ +#define OCEANBASE_OBSERVER_TABLE_OB_TENANT_TTL_MANAGER_H_ + +#include "share/table/ob_ttl_util.h" +#include "share/tablet/ob_tablet_info.h" + +namespace oceanbase +{ +namespace table +{ +class ObRootService; + +class ObTTLServerInfo; +typedef common::ObArray TTLServerInfos; +typedef common::hash::ObHashSet ServerSet; + +/** + * the task for clear ttl history task in __all_ttl_task_status_history +*/ +class ObClearTTLHistoryTask : public common::ObTimerTask +{ +public: + ObClearTTLHistoryTask() + : sql_proxy_(nullptr), + is_inited_(false), + tenant_id_(OB_INVALID_TENANT_ID) + {} + ~ObClearTTLHistoryTask() {} + int init(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy); + virtual void runTimerTask() override; + void destroy() {} + + static const int64_t OB_KV_TTL_GC_INTERVAL = 30 * 1000L * 1000L; // 30s + static const int64_t OB_KV_TTL_GC_COUNT_PER_TASK = 4096L; +private: + common::ObMySQLProxy *sql_proxy_; + bool is_inited_; + uint64_t tenant_id_; +}; + +struct ObTTLServerInfo +{ +public: + ObTTLServerInfo() : addr_(), is_responsed_(false) {} + ~ObTTLServerInfo() = default; + TO_STRING_KV(K_(addr), K_(is_responsed)); +public: + common::ObAddr addr_; + bool is_responsed_; +}; + +class ObTTLTenantTask +{ +public: + ObTTLTenantTask(uint64_t tenant_id = OB_INVALID_ID) + : tenant_id_(tenant_id), + ttl_status_(), + is_finished_(true) + {} + ~ObTTLTenantTask() {} + + void reset() { + is_finished_ = true; + ttl_status_.status_ = OB_TTL_TASK_INVALID; + } + + TO_STRING_KV(K_(tenant_id), + K_(ttl_status), + K_(is_finished)); + +public: + uint64_t tenant_id_; + common::ObTTLStatus ttl_status_; + bool is_finished_; +}; + +class ObTTLTaskScheduler : public common::ObTimerTask +{ +public: + ObTTLTaskScheduler() + : del_ten_arr_(), sql_proxy_(nullptr), is_inited_(false), periodic_launched_(false), need_reload_(true) + {} + ~ObTTLTaskScheduler() {} + + int init(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy); + + int add_ttl_task(ObTTLTaskType task_type); + + void reset_local_tenant_task(); + + // reload latest tenant task from system table + int reload_tenant_task(); + + void runTimerTask() override; + + int try_add_periodic_task(); + void set_need_reload(bool need_reload) { need_reload_ = need_reload; } +private: + virtual bool is_enable_ttl(); + + virtual int delete_task(const uint64_t tenant_id, const uint64_t task_id); + + virtual int in_active_time(bool& is_active_time); + + virtual int insert_tenant_task(ObTTLStatus& ttl_task); + + virtual int update_task_status(uint64_t task_id, + int64_t rs_new_status, + common::ObISQLClient& proxy); + virtual int fetch_ttl_task_id(uint64_t tenant_id, int64_t &new_task_id); + + int calc_next_task_state(ObTTLTaskType user_cmd_type, + ObTTLTaskStatus curr_state, + ObTTLTaskStatus &next_state); + + ObTTLTaskStatus next_status(int64_t curr); + + int add_ttl_task_internal(TRIGGER_TYPE trigger_type); + + int check_all_tablet_task(); + int check_one_tablet_task(common::ObISQLClient &sql_client, + const uint64_t table_id, + const ObTabletID tablet_id, + bool &is_finished); + int check_is_ttl_table(const ObTableSchema &table_schema, bool &is_ttl_table); + + int check_task_need_move(bool &need_move); +private: + int check_all_tabelt_finished(bool &all_finished); + int check_tablet_table_finished(common::ObIArray &pairs, bool &all_finished); + int move_all_task_to_history_table(); +private: + static const int64_t TBALE_CHECK_BATCH_SIZE = 200; + static const int64_t TBALET_CHECK_BATCH_SIZE = 1024; + static const int64_t DEFAULT_TABLE_ARRAY_SIZE = 200; + static const int64_t DEFAULT_TABLET_PAIR_SIZE = 1024; +private: + ObTTLTenantTask tenant_task_; + ObArray del_ten_arr_; + common::ObMySQLProxy *sql_proxy_; + bool is_inited_; + uint64_t tenant_id_; + + bool periodic_launched_; + + bool need_reload_; + lib::ObMutex mutex_; + ObArray tablet_table_pairs_; + const int64_t OB_TTL_TASK_RETRY_INTERVAL = 15*1000*1000; // 15s +}; + +class ObTenantTTLManager +{ +public: + static const int64_t SCHEDULE_PERIOD = 15 * 1000L * 1000L; // 15s + explicit ObTenantTTLManager() + : is_inited_(false), + clear_ttl_history_task_(), + tenant_id_(OB_INVALID_TENANT_ID), + task_scheduler_(), + tg_id_(0) + {} + + virtual ~ObTenantTTLManager() {} + int init(const uint64_t tenant_id, ObMySQLProxy &sql_proxy); + int start(); + void wait(); + void stop(); + void destroy(); + int handle_user_ttl(const obrpc::ObTTLRequestArg& arg); +private: + bool is_inited_; + ObClearTTLHistoryTask clear_ttl_history_task_; + uint64_t tenant_id_; + ObTTLTaskScheduler task_scheduler_; + int tg_id_; +}; + + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_TABLE_OB_TENANT_TTL_MANAGER_H_ */ \ No newline at end of file diff --git a/src/observer/table/ttl/ob_ttl_service.cpp b/src/observer/table/ttl/ob_ttl_service.cpp new file mode 100644 index 0000000000..ad396d71f7 --- /dev/null +++ b/src/observer/table/ttl/ob_ttl_service.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "observer/table/ttl/ob_ttl_service.h" +#include "observer/table/ttl/ob_tenant_ttl_manager.h" +using namespace oceanbase::observer; +namespace oceanbase +{ +namespace table +{ +int ObTTLService::init(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), K(tenant_id)); + } else { + tenant_id_ = tenant_id; + is_inited_ = true; + } + FLOG_INFO("ttl service: init", KR(ret), K_(tenant_id)); + return ret; +} +ObTTLService::~ObTTLService() +{ +} + +int ObTTLService::switch_to_leader() +{ + int64_t start_time_us = ObTimeUtility::current_time(); + FLOG_INFO("ttl_service: start to switch_to_leader", K_(tenant_id), K(start_time_us)); + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("fail to check_inner_stat", KR(ret), K_(tenant_id)); + } else { + if (OB_ISNULL(tenant_ttl_mgr_)) { + if (OB_FAIL(alloc_tenant_ttl_mgr())) { + LOG_WARN("fail to alloc tenant_ttl_mgr", KR(ret), K_(tenant_id)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(tenant_ttl_mgr_->start())) { + LOG_WARN("fail to start tenant_ttl_mgr", K_(tenant_id), KR(ret)); + } + } + } + const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; + FLOG_INFO("ttl_service: finish switch_to_leader", KR(ret), K_(tenant_id), K(cost_us), KP_(tenant_ttl_mgr)); + return ret; +} + +int ObTTLService::switch_to_follower_gracefully() +{ + int ret = OB_SUCCESS; + inner_switch_to_follower(); + return ret; +} + +void ObTTLService::switch_to_follower_forcedly() +{ + inner_switch_to_follower(); +} + +void ObTTLService::inner_switch_to_follower() +{ + FLOG_INFO("ttl_service: switch_to_follower", K_(tenant_id)); + const int64_t start_time_us = ObTimeUtility::current_time(); + stop(); + wait(); + const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; + FLOG_INFO("ttl_service: switch_to_follower", K_(tenant_id), K(cost_us), KP_(tenant_ttl_mgr)); +} + +int ObTTLService::launch_ttl_task(const obrpc::ObTTLRequestArg &req) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("fail to check_inner_stat", KR(ret)); + } else { + if (OB_ISNULL(tenant_ttl_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant_ttl_mgr is null", KR(ret), K_(tenant_id), KP_(tenant_ttl_mgr)); + } else if (OB_FAIL(tenant_ttl_mgr_->handle_user_ttl(req))) { + LOG_WARN("fail to handle user ttl", KR(ret), K_(tenant_id)); + } + } + return ret; +} + +void ObTTLService::stop() +{ + FLOG_INFO("ttl_service: start to stop", K_(tenant_id)); + if (OB_NOT_NULL(tenant_ttl_mgr_)) { + LOG_INFO("tenant_ttl_mgr start to stop", K_(tenant_id)); + tenant_ttl_mgr_->stop(); + } + FLOG_INFO("ttl_service: finish to stop", K_(tenant_id)); +} + +void ObTTLService::wait() +{ + FLOG_INFO("ttl_service: start to wait", K_(tenant_id)); + int ret = OB_SUCCESS; + if (OB_NOT_NULL(tenant_ttl_mgr_)) { + LOG_INFO("tenant_ttl_mgr start to wait", K_(tenant_id)); + tenant_ttl_mgr_->wait(); + } + FLOG_INFO("ttl_service: finish to wait", K_(tenant_id)); +} +void ObTTLService::destroy() +{ + FLOG_INFO("ttl_service: start to destroy", K_(tenant_id)); + delete_tenant_ttl_mgr(); + FLOG_INFO("ttl_service: finish to destroy", K_(tenant_id)); +} +int ObTTLService::alloc_tenant_ttl_mgr() +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + int64_t len = sizeof(ObTenantTTLManager); + ObMemAttr attr(MTL_ID(), "tenant_ttl_mgr"); + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("fail to check_inner_stat", KR(ret)); + } else if (OB_NOT_NULL(tenant_ttl_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant_ttl_mgr is not null", K_(tenant_id), KR(ret), KP_(tenant_ttl_mgr)); + } else if (OB_ISNULL(buf = ob_malloc(sizeof(ObTenantTTLManager), attr))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", KR(ret), K_(tenant_id), K(len)); + } else if (FALSE_IT(tenant_ttl_mgr_ = new(buf)ObTenantTTLManager())) { + } else if (OB_FAIL(tenant_ttl_mgr_->init(tenant_id_, *GCTX.sql_proxy_))) { + LOG_WARN("fail to init tenant_ttl_mgr", K_(tenant_id), KR(ret)); + } + if (OB_SUCC(ret)) { + LOG_INFO("succ to alloc tenant_ttl_mgr", K_(tenant_id), KP_(tenant_ttl_mgr)); + } else { + delete_tenant_ttl_mgr(); + } + return ret; +} + +void ObTTLService::delete_tenant_ttl_mgr() +{ + if (OB_ISNULL(tenant_ttl_mgr_)) { + // no need to delete + } else { + tenant_ttl_mgr_->destroy(); + OB_DELETE(ObTenantTTLManager, "tenant_ttl_mgr", tenant_ttl_mgr_); + tenant_ttl_mgr_ = nullptr; + } + LOG_INFO("finish to delete tenant_ttl_mgr", K_(tenant_id)); +} + +int ObTTLService::check_inner_stat() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K_(tenant_id)); + } + return ret; +} + +int ObTTLService::mtl_init(ObTTLService *&service) +{ + int ret = OB_SUCCESS; + if (service->init(MTL_ID())) { + LOG_WARN("fail to ini ttl service", KR(ret)); + } + return ret; +} +} +} diff --git a/src/observer/table/ttl/ob_ttl_service.h b/src/observer/table/ttl/ob_ttl_service.h new file mode 100644 index 0000000000..6da04643b0 --- /dev/null +++ b/src/observer/table/ttl/ob_ttl_service.h @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_TABLE_OB_TTL_SERVICE_DEFINE_H_ +#define OCEANBASE_OBSERVER_TABLE_OB_TTL_SERVICE_DEFINE_H_ +#include "share/ob_ls_id.h" +#include "logservice/ob_log_base_type.h" +#include "share/scn.h" +#include "lib/lock/ob_recursive_mutex.h" +#include "observer/table/ttl/ob_tenant_ttl_manager.h" +namespace oceanbase +{ +namespace table +{ +class ObTTLService : public logservice::ObIReplaySubHandler, + public logservice::ObICheckpointSubHandler, + public logservice::ObIRoleChangeSubHandler +{ +public: + ObTTLService() + : is_inited_(false), + tenant_id_(OB_INVALID_TENANT_ID), + tenant_ttl_mgr_(nullptr) + {} + virtual ~ObTTLService(); + int init(const uint64_t tenant_id); + int flush(share::SCN &rec_scn) + { + UNUSED(rec_scn); + return OB_SUCCESS; + } + share::SCN get_rec_scn() override { return share::SCN::max_scn(); } + + // for replay, do nothing + int replay(const void *buffer, + const int64_t buf_size, + const palf::LSN &lsn, + const share::SCN &scn) + { + UNUSED(buffer); + UNUSED(buf_size); + UNUSED(lsn); + UNUSED(scn); + return OB_SUCCESS; + } + static int mtl_init(ObTTLService *&service); + // switch leader + void switch_to_follower_forcedly(); + int switch_to_leader(); + int switch_to_follower_gracefully(); + int resume_leader() { return switch_to_leader(); } + int launch_ttl_task(const obrpc::ObTTLRequestArg &req); + uint64_t get_tenant_id() const { return tenant_id_; } + int start() { return OB_SUCCESS; } + void stop(); + void wait(); + void destroy(); +private: + int check_inner_stat(); + int alloc_tenant_ttl_mgr(); + void delete_tenant_ttl_mgr(); + void inner_switch_to_follower(); +private: + bool is_inited_; + int64_t tenant_id_; + ObTenantTTLManager *tenant_ttl_mgr_; +}; +} +} +#endif // OCEANBASE_OBSERVER_TABLE_OB_TTL_SERVICE_DEFINE_H_ diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 65b5ed0a5b..b32a7b94d2 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -2414,6 +2414,7 @@ int ObDDLService::set_raw_table_options( int ret = OB_SUCCESS; //replace alter options need_update_index_table = false; + uint64_t tenant_id = new_table_schema.get_tenant_id(); for (int32_t i = ObAlterTableArg::AUTO_INCREMENT; OB_SUCC(ret) && i < ObAlterTableArg::MAX_OPTION; ++i) { if (alter_table_schema.alter_option_bitset_.has_member(i)) { @@ -2498,7 +2499,6 @@ int ObDDLService::set_raw_table_options( const ObString &origin_table_name = alter_table_schema.get_origin_table_name(); const ObString &new_database_name = alter_table_schema.get_database_name(); const ObString &origin_database_name = alter_table_schema.get_origin_database_name(); - uint64_t tenant_id = new_table_schema.get_tenant_id(); ObNameCaseMode mode = OB_NAME_CASE_INVALID; bool is_oracle_mode = false; bool has_mv = false; @@ -2723,6 +2723,32 @@ int ObDDLService::set_raw_table_options( new_table_schema.set_encryption_str(alter_table_schema.get_encryption_str()); break; } + case ObAlterTableArg::TTL_DEFINITION: { + uint64_t compat_version = OB_INVALID_VERSION; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("get min data_version failed", K(ret), K(tenant_id)); + } else if (compat_version < DATA_VERSION_4_2_1_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl definition less than 4.2.1 not support", K(ret), K(compat_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ttl definition less than 4.2.1"); + } else if (OB_FAIL(new_table_schema.set_ttl_definition(alter_table_schema.get_ttl_definition()))) { + LOG_WARN("fail to set ttl definition", K(ret)); + } + break; + } + case ObAlterTableArg::KV_ATTRIBUTES: { + uint64_t compat_version = OB_INVALID_VERSION; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("get min data_version failed", K(ret), K(tenant_id)); + } else if (compat_version < DATA_VERSION_4_2_1_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("kv attributes less than 4.2.1 not support", K(ret), K(compat_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes less than 4.2"); + } else if (OB_FAIL(new_table_schema.set_kv_attributes(alter_table_schema.get_kv_attributes()))) { + LOG_WARN("fail to set kv attributes", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unknown option!", K(i)); diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index 0756ee8658..4b6bcf56ae 100755 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -247,6 +247,7 @@ ob_set_subtarget(ob_share common_mixed detect/ob_detect_rpc_processor.cpp detect/ob_detect_manager_utils.cpp table/ob_table_load_sql_statistics.cpp + table/ob_ttl_util.cpp ) ob_set_subtarget(ob_share tablet diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index a5b5135585..575c357bad 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -27,6 +27,7 @@ #include "sql/optimizer/ob_log_join_filter.h" #include "share/ob_encryption_util.h" #include "share/ob_resource_limit.h" +#include "share/table/ob_ttl_util.h" #include "src/observer/ob_server.h" namespace oceanbase @@ -372,6 +373,12 @@ bool ObConfigRpcChecksumChecker::check(const ObConfigItem &t) const return obrpc::get_rpc_checksum_check_level_from_string(tmp_string) != obrpc::ObRpcCheckSumCheckLevel::INVALID ; } +bool ObTTLDutyDurationChecker::check(const ObConfigItem& t) const +{ + common::ObTTLDutyDuration duty_duration; + return OB_SUCCESS == common::ObTTLUtil::parse(t.str(), duty_duration) && duty_duration.is_valid(); +} + bool ObConfigMemoryLimitChecker::check(const ObConfigItem &t) const { bool is_valid = false; diff --git a/src/share/config/ob_config_helper.h b/src/share/config/ob_config_helper.h index a24e0e1e5e..0957224e12 100644 --- a/src/share/config/ob_config_helper.h +++ b/src/share/config/ob_config_helper.h @@ -461,6 +461,17 @@ private: DISALLOW_COPY_AND_ASSIGN(ObConfigRuntimeFilterChecker); }; +class ObTTLDutyDurationChecker : public ObConfigChecker { +public: + ObTTLDutyDurationChecker() + {} + virtual ~ObTTLDutyDurationChecker(){}; + bool check(const ObConfigItem& t) const; + +private: + DISALLOW_COPY_AND_ASSIGN(ObTTLDutyDurationChecker); +}; + // config item container class ObConfigStringKey { diff --git a/src/share/inner_table/ob_inner_table_schema.101_150.cpp b/src/share/inner_table/ob_inner_table_schema.101_150.cpp index 776283bf65..dcf5427481 100644 --- a/src/share/inner_table/ob_inner_table_schema.101_150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.101_150.cpp @@ -6541,6 +6541,44 @@ int ObInnerTableSchema::all_table_history_schema(ObTableSchema &table_schema) true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj ttl_definition_default; + ttl_definition_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("ttl_definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + ttl_definition_default, + ttl_definition_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj kv_attributes_default; + kv_attributes_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("kv_attributes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + kv_attributes_default, + kv_attributes_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp b/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp index 35a19939ed..1104d52a95 100644 --- a/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp @@ -13965,6 +13965,44 @@ int ObInnerTableSchema::all_virtual_core_all_table_schema(ObTableSchema &table_s true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj ttl_definition_default; + ttl_definition_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("ttl_definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + ttl_definition_default, + ttl_definition_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj kv_attributes_default; + kv_attributes_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("kv_attributes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + kv_attributes_default, + kv_attributes_default); //default_value + } table_schema.set_index_using_type(USING_HASH); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp b/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp index fd3cd21106..093a67fd3b 100644 --- a/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp @@ -6421,6 +6421,44 @@ int ObInnerTableSchema::all_virtual_table_schema(ObTableSchema &table_schema) true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj ttl_definition_default; + ttl_definition_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("ttl_definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + ttl_definition_default, + ttl_definition_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj kv_attributes_default; + kv_attributes_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("kv_attributes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + kv_attributes_default, + kv_attributes_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); @@ -7769,6 +7807,44 @@ int ObInnerTableSchema::all_virtual_table_history_schema(ObTableSchema &table_sc true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj ttl_definition_default; + ttl_definition_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("ttl_definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + ttl_definition_default, + ttl_definition_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj kv_attributes_default; + kv_attributes_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("kv_attributes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + kv_attributes_default, + kv_attributes_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp b/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp index 14958f9918..c881e3573a 100644 --- a/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp @@ -7057,6 +7057,548 @@ int ObInnerTableSchema::all_virtual_query_response_time_schema(ObTableSchema &ta return ret; } +int ObInnerTableSchema::all_virtual_kv_ttl_task_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_KV_TTL_TASK_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_KV_TTL_TASK_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_update_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ttl_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("max_version_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("scan_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("row_key", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 2048, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ERROR_MSG_LEN, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_kv_ttl_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_update_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ttl_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("max_version_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("scan_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("row_key", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 2048, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ERROR_MSG_LEN, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_column_checksum_error_info_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp b/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp index 726aa959e7..82d9123b27 100644 --- a/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp @@ -6308,6 +6308,36 @@ int ObInnerTableSchema::all_virtual_table_real_agent_ora_schema(ObTableSchema &t false); //is_autoincrement } + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TTL_DEFINITION", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("KV_ATTRIBUTES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name ++column_id, //column_id diff --git a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp index e7a57f9c76..f912120692 100644 --- a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp @@ -6091,6 +6091,36 @@ int ObInnerTableSchema::all_virtual_core_all_table_ora_schema(ObTableSchema &tab true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TTL_DEFINITION", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("KV_ATTRIBUTES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } table_schema.set_index_using_type(USING_HASH); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.1_50.cpp b/src/share/inner_table/ob_inner_table_schema.1_50.cpp index b7aa2121d0..79ed597224 100644 --- a/src/share/inner_table/ob_inner_table_schema.1_50.cpp +++ b/src/share/inner_table/ob_inner_table_schema.1_50.cpp @@ -1511,6 +1511,44 @@ int ObInnerTableSchema::all_table_schema(ObTableSchema &table_schema) true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj ttl_definition_default; + ttl_definition_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("ttl_definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + ttl_definition_default, + ttl_definition_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj kv_attributes_default; + kv_attributes_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("kv_attributes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DEFAULT_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + kv_attributes_default, + kv_attributes_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp index 0e3460eab6..1a570f07f5 100644 --- a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp @@ -25,6 +25,106 @@ using namespace common; namespace share { +int ObInnerTableSchema::dba_ob_kv_ttl_tasks_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_KV_TTL_TASKS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_KV_TTL_TASKS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT b.table_name as TABLE_NAME, a.table_id as TABLE_ID, a.tablet_id as TABLET_ID, a.task_id as TASK_ID, usec_to_time(a.task_start_time) as START_TIME, usec_to_time(a.task_update_time) as END_TIME, case a.trigger_type when 0 then "PERIODIC" when 1 then "USER" else "INVALID" END AS TRIGGER_TYPE, case a.status when 0 then "PREPARED" when 1 then "RUNNING" when 2 then "PENDING" when 3 then "CANCELED" when 4 then "FINISHED" when 5 then "MOVED" when 15 then "RS_TRIGGERING" when 16 then "RS_SUSPENDING" when 17 then "RS_CANCELING" when 18 then "RS_MOVING" when 47 then "RS_TRIGGERD" when 48 then "RS_SUSPENDED" when 49 then "RS_CANCELED" when 50 then "RS_MOVED" else "INVALID" END AS STATUS, a.ttl_del_cnt as TTL_DEL_CNT, a.max_version_del_cnt as MAX_VERSION_DEL_CNT, a.scan_cnt as SCAN_CNT, a.ret_code as RET_CODE FROM oceanbase.__all_virtual_kv_ttl_task a left outer JOIN oceanbase.__all_table b on a.table_id = b.table_id and a.tenant_id = effective_tenant_id() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_ob_kv_ttl_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_KV_TTL_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_KV_TTL_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT b.table_name as TABLE_NAME, a.table_id as TABLE_ID, a.tablet_id as TABLET_ID, a.task_id as TASK_ID, usec_to_time(a.task_start_time) as START_TIME, usec_to_time(a.task_update_time) as END_TIME, case a.trigger_type when 0 then "PERIODIC" when 1 then "USER" else "INVALID" END AS TRIGGER_TYPE, case a.status when 0 then "PREPARED" when 1 then "RUNNING" when 2 then "PENDING" when 3 then "CANCELED" when 4 then "FINISHED" when 5 then "MOVED" else "INVALID" END AS STATUS, a.ttl_del_cnt as TTL_DEL_CNT, a.max_version_del_cnt as MAX_VERSION_DEL_CNT, a.scan_cnt as SCAN_CNT, a.ret_code as RET_CODE FROM oceanbase.__all_virtual_kv_ttl_task_history a left outer JOIN oceanbase.__all_table b on a.table_id = b.table_id and a.tenant_id = effective_tenant_id() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::gv_ob_log_stat_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; @@ -282,6 +382,106 @@ int ObInnerTableSchema::query_response_time_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::cdb_ob_kv_ttl_tasks_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_OB_KV_TTL_TASKS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_OB_KV_TTL_TASKS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT a.tenant_id as TENANT_ID, b.table_name as TABLE_NAME, a.table_id as TABLE_ID, a.tablet_id as TABLET_ID, a.task_id as TASK_ID, usec_to_time(a.task_start_time) as START_TIME, usec_to_time(a.task_update_time) as END_TIME, case a.trigger_type when 0 then "PERIODIC" when 1 then "USER" else "INVALID" END AS TRIGGER_TYPE, case a.status when 0 then "PREPARED" when 1 then "RUNNING" when 2 then "PENDING" when 3 then "CANCELED" when 4 then "FINISHED" when 5 then "MOVED" when 15 then "RS_TRIGGERING" when 16 then "RS_SUSPENDING" when 17 then "RS_CANCELING" when 18 then "RS_MOVING" when 47 then "RS_TRIGGERD" when 48 then "RS_SUSPENDED" when 49 then "RS_CANCELED" when 50 then "RS_MOVED" else "INVALID" END AS STATUS, a.ttl_del_cnt as TTL_DEL_CNT, a.max_version_del_cnt as MAX_VERSION_DEL_CNT, a.scan_cnt as SCAN_CNT, a.ret_code as RET_CODE FROM oceanbase.__all_virtual_kv_ttl_task a left outer JOIN oceanbase.__all_virtual_table b on a.table_id = b.table_id and a.tenant_id = b.tenant_id )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_ob_kv_ttl_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_OB_KV_TTL_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_OB_KV_TTL_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT a.tenant_id as TENANT_ID, b.table_name as TABLE_NAME, a.table_id as TABLE_ID, a.tablet_id as TABLET_ID, a.task_id as TASK_ID, usec_to_time(a.task_start_time) as START_TIME, usec_to_time(a.task_update_time) as END_TIME, case a.trigger_type when 0 then "PERIODIC" when 1 then "USER" else "INVALID" END AS TRIGGER_TYPE, case a.status when 0 then "PREPARED" when 1 then "RUNNING" when 2 then "PENDING" when 3 then "CANCELED" when 4 then "FINISHED" when 5 then "MOVED" else "INVALID" END AS STATUS, a.ttl_del_cnt as TTL_DEL_CNT, a.max_version_del_cnt as MAX_VERSION_DEL_CNT, a.scan_cnt as SCAN_CNT, a.ret_code as RET_CODE FROM oceanbase.__all_virtual_kv_ttl_task_history a left outer JOIN oceanbase.__all_virtual_table b on a.table_id = b.table_id and a.tenant_id = b.tenant_id )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::dba_rsrc_plans_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp index 8ad11b58e6..3676e6e20b 100644 --- a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp @@ -12155,6 +12155,251 @@ int ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_expirati return ret; } +int ObInnerTableSchema::all_kv_ttl_task_idx_kv_ttl_task_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_kv_ttl_task_history_idx_kv_ttl_task_history_upd_time_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_update_time", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_data_table_id_real_agent_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.401_450.cpp b/src/share/inner_table/ob_inner_table_schema.401_450.cpp index a3bf61f1c1..002fc90796 100644 --- a/src/share/inner_table/ob_inner_table_schema.401_450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.401_450.cpp @@ -2206,6 +2206,580 @@ int ObInnerTableSchema::all_log_restore_source_schema(ObTableSchema &table_schem return ret; } +int ObInnerTableSchema::all_kv_ttl_task_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_update_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ttl_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("max_version_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("scan_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("row_key", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 2048, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ERROR_MSG_LEN, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_kv_ttl_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_update_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ttl_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("max_version_del_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("scan_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("row_key", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 2048, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ERROR_MSG_LEN, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_service_epoch_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.50401_50450.cpp b/src/share/inner_table/ob_inner_table_schema.50401_50450.cpp index 8537cde208..2ff109c2a4 100644 --- a/src/share/inner_table/ob_inner_table_schema.50401_50450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.50401_50450.cpp @@ -1375,6 +1375,276 @@ int ObInnerTableSchema::all_log_restore_source_aux_lob_meta_schema(ObTableSchema return ret; } +int ObInnerTableSchema::all_kv_ttl_task_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_kv_ttl_task_history_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_service_epoch_aux_lob_meta_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.60401_60450.cpp b/src/share/inner_table/ob_inner_table_schema.60401_60450.cpp index 94175669c1..ddd8397f90 100644 --- a/src/share/inner_table/ob_inner_table_schema.60401_60450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.60401_60450.cpp @@ -925,6 +925,186 @@ int ObInnerTableSchema::all_log_restore_source_aux_lob_piece_schema(ObTableSchem return ret; } +int ObInnerTableSchema::all_kv_ttl_task_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_kv_ttl_task_history_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_service_epoch_aux_lob_piece_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 287cf87540..4ba8c3e8a1 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -499,6 +499,8 @@ public: static int all_mock_fk_parent_table_column_schema(share::schema::ObTableSchema &table_schema); static int all_mock_fk_parent_table_column_history_schema(share::schema::ObTableSchema &table_schema); static int all_log_restore_source_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_history_schema(share::schema::ObTableSchema &table_schema); static int all_service_epoch_schema(share::schema::ObTableSchema &table_schema); static int all_spatial_reference_systems_schema(share::schema::ObTableSchema &table_schema); static int all_column_checksum_error_info_schema(share::schema::ObTableSchema &table_schema); @@ -862,6 +864,8 @@ public: static int all_virtual_mock_fk_parent_table_column_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_log_restore_source_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_query_response_time_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_kv_ttl_task_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_kv_ttl_task_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_column_checksum_error_info_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_kvcache_handle_leak_info_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tablet_compaction_info_schema(share::schema::ObTableSchema &table_schema); @@ -1404,11 +1408,15 @@ public: static int dba_ob_deadlock_event_history_schema(share::schema::ObTableSchema &table_schema); static int cdb_ob_deadlock_event_history_schema(share::schema::ObTableSchema &table_schema); static int cdb_ob_sys_variables_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_kv_ttl_tasks_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_kv_ttl_task_history_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_log_stat_schema(share::schema::ObTableSchema &table_schema); static int v_ob_log_stat_schema(share::schema::ObTableSchema &table_schema); static int st_geometry_columns_schema(share::schema::ObTableSchema &table_schema); static int st_spatial_reference_systems_schema(share::schema::ObTableSchema &table_schema); static int query_response_time_schema(share::schema::ObTableSchema &table_schema); + static int cdb_ob_kv_ttl_tasks_schema(share::schema::ObTableSchema &table_schema); + static int cdb_ob_kv_ttl_task_history_schema(share::schema::ObTableSchema &table_schema); static int dba_rsrc_plans_schema(share::schema::ObTableSchema &table_schema); static int dba_rsrc_plan_directives_schema(share::schema::ObTableSchema &table_schema); static int dba_rsrc_group_mappings_schema(share::schema::ObTableSchema &table_schema); @@ -2144,6 +2152,8 @@ public: static int all_mock_fk_parent_table_column_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_mock_fk_parent_table_column_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_log_restore_source_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_service_epoch_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_spatial_reference_systems_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_column_checksum_error_info_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); @@ -2396,6 +2406,8 @@ public: static int all_mock_fk_parent_table_column_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_mock_fk_parent_table_column_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_log_restore_source_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_service_epoch_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_spatial_reference_systems_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_checksum_error_info_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -2622,6 +2634,8 @@ public: static int all_rls_context_history_idx_rls_context_his_table_id_schema(share::schema::ObTableSchema &table_schema); static int all_dbms_lock_allocated_idx_dbms_lock_allocated_lockhandle_schema(share::schema::ObTableSchema &table_schema); static int all_dbms_lock_allocated_idx_dbms_lock_allocated_expiration_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_idx_kv_ttl_task_table_id_schema(share::schema::ObTableSchema &table_schema); + static int all_kv_ttl_task_history_idx_kv_ttl_task_history_upd_time_schema(share::schema::ObTableSchema &table_schema); private: DISALLOW_COPY_AND_ASSIGN(ObInnerTableSchema); @@ -2850,6 +2864,8 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_mock_fk_parent_table_column_schema, ObInnerTableSchema::all_mock_fk_parent_table_column_history_schema, ObInnerTableSchema::all_log_restore_source_schema, + ObInnerTableSchema::all_kv_ttl_task_schema, + ObInnerTableSchema::all_kv_ttl_task_history_schema, ObInnerTableSchema::all_service_epoch_schema, ObInnerTableSchema::all_spatial_reference_systems_schema, ObInnerTableSchema::all_column_checksum_error_info_schema, @@ -3216,6 +3232,8 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_mock_fk_parent_table_column_history_schema, ObInnerTableSchema::all_virtual_log_restore_source_schema, ObInnerTableSchema::all_virtual_query_response_time_schema, + ObInnerTableSchema::all_virtual_kv_ttl_task_schema, + ObInnerTableSchema::all_virtual_kv_ttl_task_history_schema, ObInnerTableSchema::all_virtual_column_checksum_error_info_schema, ObInnerTableSchema::all_virtual_kvcache_handle_leak_info_schema, ObInnerTableSchema::all_virtual_tablet_compaction_info_schema, @@ -3852,11 +3870,15 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_ob_deadlock_event_history_schema, ObInnerTableSchema::cdb_ob_deadlock_event_history_schema, ObInnerTableSchema::cdb_ob_sys_variables_schema, + ObInnerTableSchema::dba_ob_kv_ttl_tasks_schema, + ObInnerTableSchema::dba_ob_kv_ttl_task_history_schema, ObInnerTableSchema::gv_ob_log_stat_schema, ObInnerTableSchema::v_ob_log_stat_schema, ObInnerTableSchema::st_geometry_columns_schema, ObInnerTableSchema::st_spatial_reference_systems_schema, ObInnerTableSchema::query_response_time_schema, + ObInnerTableSchema::cdb_ob_kv_ttl_tasks_schema, + ObInnerTableSchema::cdb_ob_kv_ttl_task_history_schema, ObInnerTableSchema::dba_rsrc_plans_schema, ObInnerTableSchema::dba_rsrc_plan_directives_schema, ObInnerTableSchema::dba_rsrc_group_mappings_schema, @@ -4481,6 +4503,8 @@ const schema_create_func sys_index_table_schema_creators [] = { ObInnerTableSchema::all_rls_context_history_idx_rls_context_his_table_id_schema, ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_lockhandle_schema, ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_expiration_schema, + ObInnerTableSchema::all_kv_ttl_task_idx_kv_ttl_task_table_id_schema, + ObInnerTableSchema::all_kv_ttl_task_history_idx_kv_ttl_task_history_upd_time_schema, NULL,}; const schema_create_func information_schema_table_schema_creators[] = { @@ -4685,6 +4709,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_TID, OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_TID, OB_ALL_LOG_RESTORE_SOURCE_TID, + OB_ALL_KV_TTL_TASK_TID, + OB_ALL_KV_TTL_TASK_HISTORY_TID, OB_ALL_SERVICE_EPOCH_TID, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID, OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_TID, @@ -4876,6 +4902,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_PRIVILEGE_TID, OB_ALL_VIRTUAL_LOG_RESTORE_SOURCE_TID, OB_ALL_VIRTUAL_QUERY_RESPONSE_TIME_TID, + OB_ALL_VIRTUAL_KV_TTL_TASK_TID, + OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TID, OB_ALL_VIRTUAL_TABLET_COMPACTION_INFO_TID, OB_ALL_VIRTUAL_LS_REPLICA_TASK_PLAN_TID, OB_ALL_VIRTUAL_SHOW_TRACE_TID, @@ -5334,6 +5362,8 @@ const uint64_t tenant_space_tables [] = { OB_DBA_OB_ARCHIVELOG_PIECE_FILES_TID, OB_DBA_OB_BACKUP_PARAMETER_TID, OB_DBA_OB_DEADLOCK_EVENT_HISTORY_TID, + OB_DBA_OB_KV_TTL_TASKS_TID, + OB_DBA_OB_KV_TTL_TASK_HISTORY_TID, OB_GV_OB_LOG_STAT_TID, OB_V_OB_LOG_STAT_TID, OB_ST_GEOMETRY_COLUMNS_TID, @@ -5917,6 +5947,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID, OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID, OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID, + OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID, + OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID, OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TID, OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TID, OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TID, @@ -6183,6 +6215,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_META_TID, OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_META_TID, OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_META_TID, + OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID, + OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID, OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TID, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_META_TID, OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_META_TID, @@ -6414,6 +6448,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_PIECE_TID, OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_PIECE_TID, + OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID, + OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TID, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_PIECE_TID, @@ -6925,6 +6961,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_TNAME, OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_TNAME, OB_ALL_LOG_RESTORE_SOURCE_TNAME, + OB_ALL_KV_TTL_TASK_TNAME, + OB_ALL_KV_TTL_TASK_HISTORY_TNAME, OB_ALL_SERVICE_EPOCH_TNAME, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TNAME, OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_TNAME, @@ -7116,6 +7154,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_PRIVILEGE_TNAME, OB_ALL_VIRTUAL_LOG_RESTORE_SOURCE_TNAME, OB_ALL_VIRTUAL_QUERY_RESPONSE_TIME_TNAME, + OB_ALL_VIRTUAL_KV_TTL_TASK_TNAME, + OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TNAME, OB_ALL_VIRTUAL_TABLET_COMPACTION_INFO_TNAME, OB_ALL_VIRTUAL_LS_REPLICA_TASK_PLAN_TNAME, OB_ALL_VIRTUAL_SHOW_TRACE_TNAME, @@ -7574,6 +7614,8 @@ const char* const tenant_space_table_names [] = { OB_DBA_OB_ARCHIVELOG_PIECE_FILES_TNAME, OB_DBA_OB_BACKUP_PARAMETER_TNAME, OB_DBA_OB_DEADLOCK_EVENT_HISTORY_TNAME, + OB_DBA_OB_KV_TTL_TASKS_TNAME, + OB_DBA_OB_KV_TTL_TASK_HISTORY_TNAME, OB_GV_OB_LOG_STAT_TNAME, OB_V_OB_LOG_STAT_TNAME, OB_ST_GEOMETRY_COLUMNS_TNAME, @@ -8157,6 +8199,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TNAME, OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TNAME, OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TNAME, + OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TNAME, + OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TNAME, OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TNAME, OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TNAME, OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TNAME, @@ -8423,6 +8467,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_META_TNAME, OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_META_TNAME, OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_META_TNAME, + OB_ALL_KV_TTL_TASK_AUX_LOB_META_TNAME, + OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TNAME, OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TNAME, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_META_TNAME, OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_META_TNAME, @@ -8654,6 +8700,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_PIECE_TNAME, OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_PIECE_TNAME, + OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TNAME, + OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TNAME, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_PIECE_TNAME, @@ -10862,6 +10910,22 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_log_restore_source_aux_lob_piece_schema }, + { + OB_ALL_KV_TTL_TASK_TID, + OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID, + OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_kv_ttl_task_aux_lob_meta_schema, + ObInnerTableSchema::all_kv_ttl_task_aux_lob_piece_schema + }, + + { + OB_ALL_KV_TTL_TASK_HISTORY_TID, + OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID, + OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_kv_ttl_task_history_aux_lob_meta_schema, + ObInnerTableSchema::all_kv_ttl_task_history_aux_lob_piece_schema + }, + { OB_ALL_SERVICE_EPOCH_TID, OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TID, @@ -11211,12 +11275,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, } const int64_t OB_CORE_TABLE_COUNT = 4; -const int64_t OB_SYS_TABLE_COUNT = 249; -const int64_t OB_VIRTUAL_TABLE_COUNT = 722; -const int64_t OB_SYS_VIEW_COUNT = 762; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 1738; +const int64_t OB_SYS_TABLE_COUNT = 251; +const int64_t OB_VIRTUAL_TABLE_COUNT = 724; +const int64_t OB_SYS_VIEW_COUNT = 766; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 1746; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1741; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1749; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.lob.cpp b/src/share/inner_table/ob_inner_table_schema.lob.cpp index 0424cb9a96..6fa98de143 100644 --- a/src/share/inner_table/ob_inner_table_schema.lob.cpp +++ b/src/share/inner_table/ob_inner_table_schema.lob.cpp @@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map; bool lob_mapping_init() { int ret = OB_SUCCESS; - if (OB_FAIL(inner_lob_map.create(252, ObModIds::OB_INNER_LOB_HASH_SET))) { + if (OB_FAIL(inner_lob_map.create(254, ObModIds::OB_INNER_LOB_HASH_SET))) { SERVER_LOG(WARN, "fail to create inner lob map", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) { diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index e1267f13ce..6fa68be352 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -241,6 +241,8 @@ const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_HISTORY_TID = 406; // "__all_mock_fk_ const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_TID = 407; // "__all_mock_fk_parent_table_column" const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_TID = 408; // "__all_mock_fk_parent_table_column_history" const uint64_t OB_ALL_LOG_RESTORE_SOURCE_TID = 409; // "__all_log_restore_source" +const uint64_t OB_ALL_KV_TTL_TASK_TID = 410; // "__all_kv_ttl_task" +const uint64_t OB_ALL_KV_TTL_TASK_HISTORY_TID = 411; // "__all_kv_ttl_task_history" const uint64_t OB_ALL_SERVICE_EPOCH_TID = 412; // "__all_service_epoch" const uint64_t OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID = 413; // "__all_spatial_reference_systems" const uint64_t OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_TID = 416; // "__all_column_checksum_error_info" @@ -604,6 +606,8 @@ const uint64_t OB_ALL_VIRTUAL_MOCK_FK_PARENT_TABLE_COLUMN_TID = 12322; // "__all const uint64_t OB_ALL_VIRTUAL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_TID = 12323; // "__all_virtual_mock_fk_parent_table_column_history" const uint64_t OB_ALL_VIRTUAL_LOG_RESTORE_SOURCE_TID = 12324; // "__all_virtual_log_restore_source" const uint64_t OB_ALL_VIRTUAL_QUERY_RESPONSE_TIME_TID = 12325; // "__all_virtual_query_response_time" +const uint64_t OB_ALL_VIRTUAL_KV_TTL_TASK_TID = 12326; // "__all_virtual_kv_ttl_task" +const uint64_t OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TID = 12327; // "__all_virtual_kv_ttl_task_history" const uint64_t OB_ALL_VIRTUAL_COLUMN_CHECKSUM_ERROR_INFO_TID = 12330; // "__all_virtual_column_checksum_error_info" const uint64_t OB_ALL_VIRTUAL_KVCACHE_HANDLE_LEAK_INFO_TID = 12331; // "__all_virtual_kvcache_handle_leak_info" const uint64_t OB_ALL_VIRTUAL_TABLET_COMPACTION_INFO_TID = 12334; // "__all_virtual_tablet_compaction_info" @@ -1146,11 +1150,15 @@ const uint64_t OB_CDB_OB_BACKUP_PARAMETER_TID = 21296; // "CDB_OB_BACKUP_PARAMET const uint64_t OB_DBA_OB_DEADLOCK_EVENT_HISTORY_TID = 21297; // "DBA_OB_DEADLOCK_EVENT_HISTORY" const uint64_t OB_CDB_OB_DEADLOCK_EVENT_HISTORY_TID = 21298; // "CDB_OB_DEADLOCK_EVENT_HISTORY" const uint64_t OB_CDB_OB_SYS_VARIABLES_TID = 21299; // "CDB_OB_SYS_VARIABLES" +const uint64_t OB_DBA_OB_KV_TTL_TASKS_TID = 21300; // "DBA_OB_KV_TTL_TASKS" +const uint64_t OB_DBA_OB_KV_TTL_TASK_HISTORY_TID = 21301; // "DBA_OB_KV_TTL_TASK_HISTORY" const uint64_t OB_GV_OB_LOG_STAT_TID = 21302; // "GV$OB_LOG_STAT" const uint64_t OB_V_OB_LOG_STAT_TID = 21303; // "V$OB_LOG_STAT" const uint64_t OB_ST_GEOMETRY_COLUMNS_TID = 21304; // "ST_GEOMETRY_COLUMNS" const uint64_t OB_ST_SPATIAL_REFERENCE_SYSTEMS_TID = 21305; // "ST_SPATIAL_REFERENCE_SYSTEMS" const uint64_t OB_QUERY_RESPONSE_TIME_TID = 21306; // "QUERY_RESPONSE_TIME" +const uint64_t OB_CDB_OB_KV_TTL_TASKS_TID = 21307; // "CDB_OB_KV_TTL_TASKS" +const uint64_t OB_CDB_OB_KV_TTL_TASK_HISTORY_TID = 21308; // "CDB_OB_KV_TTL_TASK_HISTORY" const uint64_t OB_DBA_RSRC_PLANS_TID = 21311; // "DBA_RSRC_PLANS" const uint64_t OB_DBA_RSRC_PLAN_DIRECTIVES_TID = 21312; // "DBA_RSRC_PLAN_DIRECTIVES" const uint64_t OB_DBA_RSRC_GROUP_MAPPINGS_TID = 21313; // "DBA_RSRC_GROUP_MAPPINGS" @@ -1886,6 +1894,8 @@ const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_HISTORY_AUX_LOB_META_TID = 50406; // const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_META_TID = 50407; // "__all_mock_fk_parent_table_column_aux_lob_meta" const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_META_TID = 50408; // "__all_mock_fk_parent_table_column_history_aux_lob_meta" const uint64_t OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_META_TID = 50409; // "__all_log_restore_source_aux_lob_meta" +const uint64_t OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID = 50410; // "__all_kv_ttl_task_aux_lob_meta" +const uint64_t OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID = 50411; // "__all_kv_ttl_task_history_aux_lob_meta" const uint64_t OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TID = 50412; // "__all_service_epoch_aux_lob_meta" const uint64_t OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_META_TID = 50413; // "__all_spatial_reference_systems_aux_lob_meta" const uint64_t OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_META_TID = 50416; // "__all_column_checksum_error_info_aux_lob_meta" @@ -2138,6 +2148,8 @@ const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_HISTORY_AUX_LOB_PIECE_TID = 60406; // const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_PIECE_TID = 60407; // "__all_mock_fk_parent_table_column_aux_lob_piece" const uint64_t OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_PIECE_TID = 60408; // "__all_mock_fk_parent_table_column_history_aux_lob_piece" const uint64_t OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_PIECE_TID = 60409; // "__all_log_restore_source_aux_lob_piece" +const uint64_t OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID = 60410; // "__all_kv_ttl_task_aux_lob_piece" +const uint64_t OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID = 60411; // "__all_kv_ttl_task_history_aux_lob_piece" const uint64_t OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TID = 60412; // "__all_service_epoch_aux_lob_piece" const uint64_t OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_PIECE_TID = 60413; // "__all_spatial_reference_systems_aux_lob_piece" const uint64_t OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_PIECE_TID = 60416; // "__all_column_checksum_error_info_aux_lob_piece" @@ -2292,6 +2304,8 @@ const uint64_t OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID = 101087; // "__a const uint64_t OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID = 101088; // "__all_rls_context_history" const uint64_t OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID = 101090; // "__all_dbms_lock_allocated" const uint64_t OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID = 101091; // "__all_dbms_lock_allocated" +const uint64_t OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID = 101093; // "__all_kv_ttl_task" +const uint64_t OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID = 101094; // "__all_kv_ttl_task_history" const uint64_t OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TID = 15306; // "ALL_VIRTUAL_TABLE_REAL_AGENT_ORA" const uint64_t OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TID = 15307; // "ALL_VIRTUAL_TABLE_REAL_AGENT_ORA" const uint64_t OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TID = 15308; // "ALL_VIRTUAL_TABLE_REAL_AGENT_ORA" @@ -2579,6 +2593,8 @@ const char *const OB_ALL_MOCK_FK_PARENT_TABLE_HISTORY_TNAME = "__all_mock_fk_par const char *const OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_TNAME = "__all_mock_fk_parent_table_column"; const char *const OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_TNAME = "__all_mock_fk_parent_table_column_history"; const char *const OB_ALL_LOG_RESTORE_SOURCE_TNAME = "__all_log_restore_source"; +const char *const OB_ALL_KV_TTL_TASK_TNAME = "__all_kv_ttl_task"; +const char *const OB_ALL_KV_TTL_TASK_HISTORY_TNAME = "__all_kv_ttl_task_history"; const char *const OB_ALL_SERVICE_EPOCH_TNAME = "__all_service_epoch"; const char *const OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TNAME = "__all_spatial_reference_systems"; const char *const OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_TNAME = "__all_column_checksum_error_info"; @@ -2942,6 +2958,8 @@ const char *const OB_ALL_VIRTUAL_MOCK_FK_PARENT_TABLE_COLUMN_TNAME = "__all_virt const char *const OB_ALL_VIRTUAL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_TNAME = "__all_virtual_mock_fk_parent_table_column_history"; const char *const OB_ALL_VIRTUAL_LOG_RESTORE_SOURCE_TNAME = "__all_virtual_log_restore_source"; const char *const OB_ALL_VIRTUAL_QUERY_RESPONSE_TIME_TNAME = "__all_virtual_query_response_time"; +const char *const OB_ALL_VIRTUAL_KV_TTL_TASK_TNAME = "__all_virtual_kv_ttl_task"; +const char *const OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TNAME = "__all_virtual_kv_ttl_task_history"; const char *const OB_ALL_VIRTUAL_COLUMN_CHECKSUM_ERROR_INFO_TNAME = "__all_virtual_column_checksum_error_info"; const char *const OB_ALL_VIRTUAL_KVCACHE_HANDLE_LEAK_INFO_TNAME = "__all_virtual_kvcache_handle_leak_info"; const char *const OB_ALL_VIRTUAL_TABLET_COMPACTION_INFO_TNAME = "__all_virtual_tablet_compaction_info"; @@ -3484,11 +3502,15 @@ const char *const OB_CDB_OB_BACKUP_PARAMETER_TNAME = "CDB_OB_BACKUP_PARAMETER"; const char *const OB_DBA_OB_DEADLOCK_EVENT_HISTORY_TNAME = "DBA_OB_DEADLOCK_EVENT_HISTORY"; const char *const OB_CDB_OB_DEADLOCK_EVENT_HISTORY_TNAME = "CDB_OB_DEADLOCK_EVENT_HISTORY"; const char *const OB_CDB_OB_SYS_VARIABLES_TNAME = "CDB_OB_SYS_VARIABLES"; +const char *const OB_DBA_OB_KV_TTL_TASKS_TNAME = "DBA_OB_KV_TTL_TASKS"; +const char *const OB_DBA_OB_KV_TTL_TASK_HISTORY_TNAME = "DBA_OB_KV_TTL_TASK_HISTORY"; const char *const OB_GV_OB_LOG_STAT_TNAME = "GV$OB_LOG_STAT"; const char *const OB_V_OB_LOG_STAT_TNAME = "V$OB_LOG_STAT"; const char *const OB_ST_GEOMETRY_COLUMNS_TNAME = "ST_GEOMETRY_COLUMNS"; const char *const OB_ST_SPATIAL_REFERENCE_SYSTEMS_TNAME = "ST_SPATIAL_REFERENCE_SYSTEMS"; const char *const OB_QUERY_RESPONSE_TIME_TNAME = "QUERY_RESPONSE_TIME"; +const char *const OB_CDB_OB_KV_TTL_TASKS_TNAME = "CDB_OB_KV_TTL_TASKS"; +const char *const OB_CDB_OB_KV_TTL_TASK_HISTORY_TNAME = "CDB_OB_KV_TTL_TASK_HISTORY"; const char *const OB_DBA_RSRC_PLANS_TNAME = "DBA_RSRC_PLANS"; const char *const OB_DBA_RSRC_PLAN_DIRECTIVES_TNAME = "DBA_RSRC_PLAN_DIRECTIVES"; const char *const OB_DBA_RSRC_GROUP_MAPPINGS_TNAME = "DBA_RSRC_GROUP_MAPPINGS"; @@ -4224,6 +4246,8 @@ const char *const OB_ALL_MOCK_FK_PARENT_TABLE_HISTORY_AUX_LOB_META_TNAME = "__al const char *const OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_META_TNAME = "__all_mock_fk_parent_table_column_aux_lob_meta"; const char *const OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_META_TNAME = "__all_mock_fk_parent_table_column_history_aux_lob_meta"; const char *const OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_META_TNAME = "__all_log_restore_source_aux_lob_meta"; +const char *const OB_ALL_KV_TTL_TASK_AUX_LOB_META_TNAME = "__all_kv_ttl_task_aux_lob_meta"; +const char *const OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TNAME = "__all_kv_ttl_task_history_aux_lob_meta"; const char *const OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TNAME = "__all_service_epoch_aux_lob_meta"; const char *const OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_META_TNAME = "__all_spatial_reference_systems_aux_lob_meta"; const char *const OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_META_TNAME = "__all_column_checksum_error_info_aux_lob_meta"; @@ -4476,6 +4500,8 @@ const char *const OB_ALL_MOCK_FK_PARENT_TABLE_HISTORY_AUX_LOB_PIECE_TNAME = "__a const char *const OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_AUX_LOB_PIECE_TNAME = "__all_mock_fk_parent_table_column_aux_lob_piece"; const char *const OB_ALL_MOCK_FK_PARENT_TABLE_COLUMN_HISTORY_AUX_LOB_PIECE_TNAME = "__all_mock_fk_parent_table_column_history_aux_lob_piece"; const char *const OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_PIECE_TNAME = "__all_log_restore_source_aux_lob_piece"; +const char *const OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TNAME = "__all_kv_ttl_task_aux_lob_piece"; +const char *const OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TNAME = "__all_kv_ttl_task_history_aux_lob_piece"; const char *const OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TNAME = "__all_service_epoch_aux_lob_piece"; const char *const OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_PIECE_TNAME = "__all_spatial_reference_systems_aux_lob_piece"; const char *const OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_PIECE_TNAME = "__all_column_checksum_error_info_aux_lob_piece"; @@ -4630,6 +4656,8 @@ const char *const OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TNAME = "__idx_439 const char *const OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TNAME = "__idx_440_idx_rls_context_his_table_id"; const char *const OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TNAME = "__idx_471_idx_dbms_lock_allocated_lockhandle"; const char *const OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TNAME = "__idx_471_idx_dbms_lock_allocated_expiration"; +const char *const OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TNAME = "__idx_410_idx_kv_ttl_task_table_id"; +const char *const OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TNAME = "__idx_411_idx_kv_ttl_task_history_upd_time"; const char *const OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TNAME = "__idx_15120_idx_data_table_id_real_agent"; const char *const OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TNAME = "__idx_15120_idx_db_tb_name_real_agent"; const char *const OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TNAME = "__idx_15120_idx_tb_name_real_agent"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index d76226e66a..f774c0f205 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -273,6 +273,8 @@ all_table_def = dict( ('external_file_location_access_info', 'varbinary:OB_MAX_VARCHAR_LENGTH', 'true'), ('external_file_format', 'varbinary:OB_MAX_VARCHAR_LENGTH', 'true'), ('external_file_pattern', 'varbinary:OB_MAX_VARCHAR_LENGTH', 'true'), + ('ttl_definition', 'varchar:OB_MAX_DEFAULT_VALUE_LENGTH', 'false', ''), + ('kv_attributes', 'varchar:OB_MAX_DEFAULT_VALUE_LENGTH', 'false', '') ], ) @@ -5096,8 +5098,63 @@ def_table_schema( ], ) -# 410 : __all_kv_ttl_task -# 411 : __all_kv_ttl_task_history +all_kv_ttl_task_def = dict( + owner = 'shenyunlong.syl', + table_name = '__all_kv_ttl_task', + table_id = '410', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('task_id', 'int'), + ('table_id', 'int'), + ('tablet_id', 'int') + ], + in_tenant_space = True, + is_cluster_private = True, + meta_record_in_sys = False, + normal_columns = [ + ('task_start_time', 'int'), + ('task_update_time', 'int'), + ('trigger_type', 'int'), + ('status', 'int'), + ('ttl_del_cnt', 'int'), + ('max_version_del_cnt', 'int'), + ('scan_cnt', 'int'), + ('row_key', 'varbinary:2048'), + ('ret_code', 'varchar:OB_MAX_ERROR_MSG_LEN') + ], +) + +all_kv_ttl_task_history_def = dict( + owner = 'shenyunlong.syl', + table_name = '__all_kv_ttl_task_history', + table_id = '411', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('task_id', 'int'), + ('table_id', 'int'), + ('tablet_id', 'int') + ], + in_tenant_space = True, + is_cluster_private = True, + meta_record_in_sys = False, + normal_columns = [ + ('task_start_time', 'int'), + ('task_update_time', 'int'), + ('trigger_type', 'int'), + ('status', 'int'), + ('ttl_del_cnt', 'int'), + ('max_version_del_cnt', 'int'), + ('scan_cnt', 'int'), + ('row_key', 'varbinary:2048'), + ('ret_code', 'varchar:OB_MAX_ERROR_MSG_LEN') + ], +) +def_table_schema(**all_kv_ttl_task_def) +def_table_schema(**all_kv_ttl_task_history_def) def_table_schema( owner = 'donglou.zl', @@ -11544,8 +11601,17 @@ def_table_schema( vtable_route_policy = 'distributed', ) -# 12326: __all_virtual_kv_ttl_task -# 12327: __all_virtual_kv_ttl_task_history +def_table_schema(**gen_iterate_private_virtual_table_def( + table_id = '12326', + table_name = '__all_virtual_kv_ttl_task', + keywords = all_def_keywords['__all_kv_ttl_task'], + in_tenant_space=True)) + +def_table_schema(**gen_iterate_private_virtual_table_def( + table_id = '12327', + table_name = '__all_virtual_kv_ttl_task_history', + keywords = all_def_keywords['__all_kv_ttl_task_history'], + in_tenant_space=True)) # 12328: __all_virtual_tenant_datafile # 12329: __all_virtual_tenant_datafile_history @@ -24362,8 +24428,89 @@ def_table_schema( """.replace("\n", " "), ) -# 21300: DBA_OB_KV_TTL_TASKS -# 21301: DBA_OB_KV_TTL_TASK_HISTORY +def_table_schema( + owner = 'shenyunlong.syl', + table_name = 'DBA_OB_KV_TTL_TASKS', + table_id = '21300', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + b.table_name as TABLE_NAME, + a.table_id as TABLE_ID, + a.tablet_id as TABLET_ID, + a.task_id as TASK_ID, + usec_to_time(a.task_start_time) as START_TIME, + usec_to_time(a.task_update_time) as END_TIME, + case a.trigger_type + when 0 then "PERIODIC" + when 1 then "USER" + else "INVALID" END AS TRIGGER_TYPE, + case a.status + when 0 then "PREPARED" + when 1 then "RUNNING" + when 2 then "PENDING" + when 3 then "CANCELED" + when 4 then "FINISHED" + when 5 then "MOVED" + when 15 then "RS_TRIGGERING" + when 16 then "RS_SUSPENDING" + when 17 then "RS_CANCELING" + when 18 then "RS_MOVING" + when 47 then "RS_TRIGGERD" + when 48 then "RS_SUSPENDED" + when 49 then "RS_CANCELED" + when 50 then "RS_MOVED" + else "INVALID" END AS STATUS, + a.ttl_del_cnt as TTL_DEL_CNT, + a.max_version_del_cnt as MAX_VERSION_DEL_CNT, + a.scan_cnt as SCAN_CNT, + a.ret_code as RET_CODE + FROM oceanbase.__all_virtual_kv_ttl_task a left outer JOIN oceanbase.__all_table b on + a.table_id = b.table_id and a.tenant_id = effective_tenant_id() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'shenyunlong.syl', + table_name = 'DBA_OB_KV_TTL_TASK_HISTORY', + table_id = '21301', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + b.table_name as TABLE_NAME, + a.table_id as TABLE_ID, + a.tablet_id as TABLET_ID, + a.task_id as TASK_ID, + usec_to_time(a.task_start_time) as START_TIME, + usec_to_time(a.task_update_time) as END_TIME, + case a.trigger_type + when 0 then "PERIODIC" + when 1 then "USER" + else "INVALID" END AS TRIGGER_TYPE, + case a.status + when 0 then "PREPARED" + when 1 then "RUNNING" + when 2 then "PENDING" + when 3 then "CANCELED" + when 4 then "FINISHED" + when 5 then "MOVED" + else "INVALID" END AS STATUS, + a.ttl_del_cnt as TTL_DEL_CNT, + a.max_version_del_cnt as MAX_VERSION_DEL_CNT, + a.scan_cnt as SCAN_CNT, + a.ret_code as RET_CODE + FROM oceanbase.__all_virtual_kv_ttl_task_history a left outer JOIN oceanbase.__all_table b on + a.table_id = b.table_id and a.tenant_id = effective_tenant_id() +""".replace("\n", " ") +) def_table_schema( owner = 'xianlin.lh', @@ -24497,8 +24644,89 @@ def_table_schema( """.replace("\n", " "), ) -# 21307: CDB_OB_KV_TTL_TASKS -# 21308: CDB_OB_KV_TTL_TASK_HISTORY +def_table_schema( + owner = 'shenyunlong.syl', + table_name = 'CDB_OB_KV_TTL_TASKS', + table_id = '21307', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + a.tenant_id as TENANT_ID, + b.table_name as TABLE_NAME, + a.table_id as TABLE_ID, + a.tablet_id as TABLET_ID, + a.task_id as TASK_ID, + usec_to_time(a.task_start_time) as START_TIME, + usec_to_time(a.task_update_time) as END_TIME, + case a.trigger_type + when 0 then "PERIODIC" + when 1 then "USER" + else "INVALID" END AS TRIGGER_TYPE, + case a.status + when 0 then "PREPARED" + when 1 then "RUNNING" + when 2 then "PENDING" + when 3 then "CANCELED" + when 4 then "FINISHED" + when 5 then "MOVED" + when 15 then "RS_TRIGGERING" + when 16 then "RS_SUSPENDING" + when 17 then "RS_CANCELING" + when 18 then "RS_MOVING" + when 47 then "RS_TRIGGERD" + when 48 then "RS_SUSPENDED" + when 49 then "RS_CANCELED" + when 50 then "RS_MOVED" + else "INVALID" END AS STATUS, + a.ttl_del_cnt as TTL_DEL_CNT, + a.max_version_del_cnt as MAX_VERSION_DEL_CNT, + a.scan_cnt as SCAN_CNT, + a.ret_code as RET_CODE + FROM oceanbase.__all_virtual_kv_ttl_task a left outer JOIN oceanbase.__all_virtual_table b on + a.table_id = b.table_id and a.tenant_id = b.tenant_id +""".replace("\n", " ") +) + +def_table_schema( + owner = 'shenyunlong.syl', + table_name = 'CDB_OB_KV_TTL_TASK_HISTORY', + table_id = '21308', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + a.tenant_id as TENANT_ID, + b.table_name as TABLE_NAME, + a.table_id as TABLE_ID, + a.tablet_id as TABLET_ID, + a.task_id as TASK_ID, + usec_to_time(a.task_start_time) as START_TIME, + usec_to_time(a.task_update_time) as END_TIME, + case a.trigger_type + when 0 then "PERIODIC" + when 1 then "USER" + else "INVALID" END AS TRIGGER_TYPE, + case a.status + when 0 then "PREPARED" + when 1 then "RUNNING" + when 2 then "PENDING" + when 3 then "CANCELED" + when 4 then "FINISHED" + when 5 then "MOVED" + else "INVALID" END AS STATUS, + a.ttl_del_cnt as TTL_DEL_CNT, + a.max_version_del_cnt as MAX_VERSION_DEL_CNT, + a.scan_cnt as SCAN_CNT, + a.ret_code as RET_CODE + FROM oceanbase.__all_virtual_kv_ttl_task_history a left outer JOIN oceanbase.__all_virtual_table b on + a.table_id = b.table_id and a.tenant_id = b.tenant_id +""".replace("\n", " ") +) # 21309: CDB_OB_DATAFILE # 21310: DBA_OB_DATAFILE @@ -53807,6 +54035,22 @@ def_sys_index_table( # 101093 : placeholder for index of __all_kv_ttl_task # 101094 : placeholder for index of __all_kv_ttl_task_history +def_sys_index_table( + index_name = 'idx_kv_ttl_task_table_id', + index_table_id = 101093, + index_columns = ['table_id'], + index_using_type = 'USING_BTREE', + index_type = 'INDEX_TYPE_NORMAL_LOCAL', + keywords = all_def_keywords['__all_kv_ttl_task']) + +def_sys_index_table( + index_name = 'idx_kv_ttl_task_history_upd_time', + index_table_id = 101094, + index_columns = ['task_update_time'], + index_using_type = 'USING_BTREE', + index_type = 'INDEX_TYPE_NORMAL_LOCAL', + keywords = all_def_keywords['__all_kv_ttl_task_history']) + ################################################################################ # Oracle Agent table Index def_agent_index_table( diff --git a/src/share/inner_table/ob_inner_table_schema_misc.ipp b/src/share/inner_table/ob_inner_table_schema_misc.ipp index 67214597f8..0858d0ed13 100644 --- a/src/share/inner_table/ob_inner_table_schema_misc.ipp +++ b/src/share/inner_table/ob_inner_table_schema_misc.ipp @@ -455,6 +455,8 @@ case OB_ALL_VIRTUAL_COLUMN_CHECKSUM_ERROR_INFO_TID: case OB_ALL_VIRTUAL_DEADLOCK_EVENT_HISTORY_TID: case OB_ALL_VIRTUAL_GLOBAL_CONTEXT_VALUE_TID: case OB_ALL_VIRTUAL_GLOBAL_TRANSACTION_TID: +case OB_ALL_VIRTUAL_KV_TTL_TASK_TID: +case OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TID: case OB_ALL_VIRTUAL_LOG_ARCHIVE_DEST_PARAMETER_TID: case OB_ALL_VIRTUAL_LOG_ARCHIVE_HISTORY_TID: case OB_ALL_VIRTUAL_LOG_ARCHIVE_PIECE_FILES_TID: @@ -946,6 +948,38 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: break; } + case OB_ALL_VIRTUAL_KV_TTL_TASK_TID: { + ObIteratePrivateVirtualTable *iter = NULL; + const bool meta_record_in_sys = false; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIteratePrivateVirtualTable, iter))) { + SERVER_LOG(WARN, "create iterate private virtual table iterator failed", KR(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_KV_TTL_TASK_TID, meta_record_in_sys, index_schema, params))) { + SERVER_LOG(WARN, "iterate private virtual table iter init failed", KR(ret)); + iter->~ObIteratePrivateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + + case OB_ALL_VIRTUAL_KV_TTL_TASK_HISTORY_TID: { + ObIteratePrivateVirtualTable *iter = NULL; + const bool meta_record_in_sys = false; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIteratePrivateVirtualTable, iter))) { + SERVER_LOG(WARN, "create iterate private virtual table iterator failed", KR(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_KV_TTL_TASK_HISTORY_TID, meta_record_in_sys, index_schema, params))) { + SERVER_LOG(WARN, "iterate private virtual table iter init failed", KR(ret)); + iter->~ObIteratePrivateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + case OB_ALL_VIRTUAL_LOG_ARCHIVE_DEST_PARAMETER_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1105,7 +1139,9 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_LS_RECOVERY_STAT_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = true; @@ -1137,9 +1173,7 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_LS_RESTORE_HISTORY_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1427,7 +1461,9 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_WR_SNAPSHOT_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1459,9 +1495,7 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_WR_SYSSTAT_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -4009,6 +4043,14 @@ case OB_ALL_DEADLOCK_EVENT_HISTORY_AUX_LOB_PIECE_TID: case OB_ALL_GLOBAL_CONTEXT_VALUE_TID: case OB_ALL_GLOBAL_CONTEXT_VALUE_AUX_LOB_META_TID: case OB_ALL_GLOBAL_CONTEXT_VALUE_AUX_LOB_PIECE_TID: +case OB_ALL_KV_TTL_TASK_TID: +case OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID: +case OB_ALL_KV_TTL_TASK_AUX_LOB_META_TID: +case OB_ALL_KV_TTL_TASK_AUX_LOB_PIECE_TID: +case OB_ALL_KV_TTL_TASK_HISTORY_TID: +case OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID: +case OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_META_TID: +case OB_ALL_KV_TTL_TASK_HISTORY_AUX_LOB_PIECE_TID: case OB_ALL_LOG_ARCHIVE_DEST_PARAMETER_TID: case OB_ALL_LOG_ARCHIVE_DEST_PARAMETER_AUX_LOB_META_TID: case OB_ALL_LOG_ARCHIVE_DEST_PARAMETER_AUX_LOB_PIECE_TID: @@ -4230,6 +4272,8 @@ case OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID: case OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID: case OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID: case OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID: +case OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID: +case OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID: #endif @@ -4263,6 +4307,7 @@ case OB_ALL_DATABASE_PRIVILEGE_TID: case OB_ALL_TENANT_OLS_LABEL_TID: case OB_ALL_TENANT_KEYSTORE_TID: case OB_ALL_COLUMN_TID: +case OB_ALL_KV_TTL_TASK_TID: case OB_ALL_RLS_CONTEXT_TID: case OB_ALL_BACKUP_SET_FILES_TID: case OB_ALL_SEQUENCE_OBJECT_TID: @@ -4302,6 +4347,7 @@ case OB_ALL_SYNONYM_TID: case OB_ALL_TENANT_SECURITY_AUDIT_TID: case OB_ALL_ROOTSERVICE_JOB_TID: case OB_ALL_PENDING_TRANSACTION_TID: +case OB_ALL_KV_TTL_TASK_HISTORY_TID: case OB_ALL_FOREIGN_KEY_TID: #endif @@ -4516,6 +4562,12 @@ case OB_ALL_COLUMN_TID: { } break; } +case OB_ALL_KV_TTL_TASK_TID: { + if (FAILEDx(index_tids.push_back(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID))) { + LOG_WARN("fail to push back index tid", KR(ret)); + } + break; +} case OB_ALL_RLS_CONTEXT_TID: { if (FAILEDx(index_tids.push_back(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID))) { LOG_WARN("fail to push back index tid", KR(ret)); @@ -4786,6 +4838,12 @@ case OB_ALL_PENDING_TRANSACTION_TID: { } break; } +case OB_ALL_KV_TTL_TASK_HISTORY_TID: { + if (FAILEDx(index_tids.push_back(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID))) { + LOG_WARN("fail to push back index tid", KR(ret)); + } + break; +} case OB_ALL_FOREIGN_KEY_TID: { if (FAILEDx(index_tids.push_back(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TID))) { LOG_WARN("fail to push back index tid", KR(ret)); @@ -5137,6 +5195,15 @@ case OB_ALL_COLUMN_TID: { } break; } +case OB_ALL_KV_TTL_TASK_TID: { + index_schema.reset(); + if (FAILEDx(ObInnerTableSchema::all_kv_ttl_task_idx_kv_ttl_task_table_id_schema(index_schema))) { + LOG_WARN("fail to create index schema", KR(ret), K(tenant_id), K(data_table_id)); + } else if (OB_FAIL(append_table_(tenant_id, index_schema, tables))) { + LOG_WARN("fail to append", KR(ret), K(tenant_id), K(data_table_id)); + } + break; +} case OB_ALL_RLS_CONTEXT_TID: { index_schema.reset(); if (FAILEDx(ObInnerTableSchema::all_rls_context_idx_rls_context_table_id_schema(index_schema))) { @@ -5560,6 +5627,15 @@ case OB_ALL_PENDING_TRANSACTION_TID: { } break; } +case OB_ALL_KV_TTL_TASK_HISTORY_TID: { + index_schema.reset(); + if (FAILEDx(ObInnerTableSchema::all_kv_ttl_task_history_idx_kv_ttl_task_history_upd_time_schema(index_schema))) { + LOG_WARN("fail to create index schema", KR(ret), K(tenant_id), K(data_table_id)); + } else if (OB_FAIL(append_table_(tenant_id, index_schema, tables))) { + LOG_WARN("fail to append", KR(ret), K(tenant_id), K(data_table_id)); + } + break; +} case OB_ALL_FOREIGN_KEY_TID: { index_schema.reset(); if (FAILEDx(ObInnerTableSchema::all_foreign_key_idx_fk_child_tid_schema(index_schema))) { @@ -5779,5 +5855,9 @@ case OB_ALL_FOREIGN_KEY_TID: { LOG_WARN("add index id failed", KR(ret), K(tenant_id)); } else if (OB_FAIL(table_ids.push_back(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID))) { LOG_WARN("add index id failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(table_ids.push_back(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID))) { + LOG_WARN("add index id failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(table_ids.push_back(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID))) { + LOG_WARN("add index id failed", KR(ret), K(tenant_id)); #endif diff --git a/src/share/ob_max_id_fetcher.cpp b/src/share/ob_max_id_fetcher.cpp index 3a50648ae5..9f3bdc2577 100755 --- a/src/share/ob_max_id_fetcher.cpp +++ b/src/share/ob_max_id_fetcher.cpp @@ -51,6 +51,7 @@ const char *ObMaxIdFetcher::max_id_name_info_[OB_MAX_ID_TYPE][2] = { { "ob_max_used_object_id", "max used object id"}, { "ob_max_used_lock_owner_id", "max used lock owner id"}, { "ob_max_used_rewrite_rule_version", "max used rewrite rule version"}, + { "ob_max_used_ttl_task_id", "max used ttl task id"}, /* the following id_type will be changed to ob_max_used_object_id and won't be persisted. */ { "ob_max_used_table_id", "max used table id"}, { "ob_max_used_database_id", "max used database id"}, @@ -114,7 +115,8 @@ int ObMaxIdFetcher::convert_id_type( case OB_MAX_USED_SYS_PL_OBJECT_ID_TYPE: case OB_MAX_USED_OBJECT_ID_TYPE: case OB_MAX_USED_LOCK_OWNER_ID_TYPE: - case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: { + case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: + case OB_MAX_USED_TTL_TASK_ID_TYPE: { dst = src; break; } @@ -304,7 +306,8 @@ int ObMaxIdFetcher::fetch_new_max_id(const uint64_t tenant_id, case OB_MAX_USED_LOCK_OWNER_ID_TYPE: case OB_MAX_USED_LS_ID_TYPE: case OB_MAX_USED_LS_GROUP_ID_TYPE: - case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: { + case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: + case OB_MAX_USED_TTL_TASK_ID_TYPE: { // won't check other id break; } diff --git a/src/share/ob_max_id_fetcher.h b/src/share/ob_max_id_fetcher.h index a6d9e4b985..33f61e4971 100755 --- a/src/share/ob_max_id_fetcher.h +++ b/src/share/ob_max_id_fetcher.h @@ -43,6 +43,7 @@ enum ObMaxIdType OB_MAX_USED_OBJECT_ID_TYPE, /* used for all kinds of user schema objects */ OB_MAX_USED_LOCK_OWNER_ID_TYPE, OB_MAX_USED_REWRITE_RULE_VERSION_TYPE, + OB_MAX_USED_TTL_TASK_ID_TYPE, /* the following ObMaxIdType will be changed to OB_MAX_USED_OBJECT_ID_TYPE and won't be persisted. */ OB_MAX_USED_TABLE_ID_TYPE, diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 973b371f62..51296f10dc 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -9231,6 +9231,41 @@ int ObBroadcastConsensusVersionArg::assign(const ObBroadcastConsensusVersionArg } return ret; } +OB_SERIALIZE_MEMBER(ObTTLResponseArg, tenant_id_, task_id_, server_addr_, task_status_); +ObTTLResponseArg::ObTTLResponseArg() + : tenant_id_(0), + task_id_(OB_INVALID_ID), + server_addr_(), + task_status_(15), + err_code_(OB_SUCCESS) +{} + +int ObTTLResponseArg::assign(const ObTTLResponseArg &other) +{ + int ret = OB_SUCCESS; + if (this == &other) { + } else { + tenant_id_ = other.tenant_id_; + task_id_ = other.task_id_; + server_addr_ = other.server_addr_; + task_status_ = other.task_status_; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObTTLRequestArg, cmd_code_, trigger_type_, task_id_, tenant_id_); + +int ObTTLRequestArg::assign(const ObTTLRequestArg &other) +{ + int ret = OB_SUCCESS; + + cmd_code_ = other.cmd_code_; + task_id_ = other.task_id_; + tenant_id_ = other.tenant_id_; + trigger_type_ = other.trigger_type_; + + return ret; +} OB_SERIALIZE_MEMBER(ObBroadcastConsensusVersionRes, ret_); diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 72aaf5be0c..c3f78a0bc5 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -1894,6 +1894,8 @@ public: TABLE_DOP, INCREMENT_MODE, ENABLE_EXTENDED_ROWID, + TTL_DEFINITION, + KV_ATTRIBUTES, MAX_OPTION = 1000 }; enum AlterPartitionType @@ -9964,6 +9966,50 @@ public: uint64_t load_count_; }; +struct ObTTLRequestArg final +{ + OB_UNIS_VERSION(1); +public: + enum TTLRequestType { + TTL_TRIGGER_TYPE = 0, + TTL_SUSPEND_TYPE = 1, + TTL_RESUME_TYPE = 2, + TTL_CANCEL_TYPE = 3, + TTL_MOVE_TYPE = 4, + TTL_INVALID_TYPE = 5 + }; + + ObTTLRequestArg() + : cmd_code_(-1), trigger_type_(-1), task_id_(OB_INVALID_ID), tenant_id_(OB_INVALID_ID) + {} + ~ObTTLRequestArg() = default; + bool is_valid() const { + // return cmd_code_ != -1 && OB_INVALID_ID != task_id_ && trigger_type_ != -1 && tenant_id_ != OB_INVALID_ID; + return cmd_code_ != -1 && trigger_type_ != -1 && tenant_id_ != OB_INVALID_ID; + } + int assign(const ObTTLRequestArg &other); + TO_STRING_KV(K_(cmd_code), K_(trigger_type), K_(task_id), K_(tenant_id)); +public: + int32_t cmd_code_; // enum TTLCmdType + int32_t trigger_type_; // system or user + int64_t task_id_; // task id + uint64_t tenant_id_; // tenand_id array +}; + +struct ObTTLResponseArg { + OB_UNIS_VERSION(1); + +public: + ObTTLResponseArg(); + int assign(const ObTTLResponseArg &other); + TO_STRING_KV(K_(tenant_id), K_(task_id), K_(server_addr), K_(task_status)); +public: + uint64_t tenant_id_; + int64_t task_id_; + ObAddr server_addr_; + uint8_t task_status_; + int err_code_; +}; struct ObAdminUnlockMemberListOpArg final { OB_UNIS_VERSION(1); diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index 8609fd7308..1606d80dbb 100755 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -235,6 +235,7 @@ public: RPC_AP(PR5 update_tenant_info_cache, OB_UPDATE_TENANT_INFO_CACHE, (obrpc::ObUpdateTenantInfoCacheArg), obrpc::ObUpdateTenantInfoCacheRes); RPC_AP(PR5 broadcast_consensus_version, OB_BROADCAST_CONSENSUS_VERSION, (obrpc::ObBroadcastConsensusVersionArg), obrpc::ObBroadcastConsensusVersionRes); RPC_S(PR5 direct_load_control, OB_DIRECT_LOAD_CONTROL, (observer::ObDirectLoadControlRequest), observer::ObDirectLoadControlResult); + RPC_S(PR5 dispatch_ttl, OB_TABLE_TTL, (obrpc::ObTTLRequestArg), obrpc::ObTTLResponseArg); RPC_S(PR5 admin_unlock_member_list_op, OB_HA_UNLOCK_MEMBER_LIST, (obrpc::ObAdminUnlockMemberListOpArg)); }; // end of class ObSrvRpcProxy diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h index 0e6296704e..2bc493c9ef 100755 --- a/src/share/ob_thread_define.h +++ b/src/share/ob_thread_define.h @@ -164,5 +164,7 @@ TG_DEF(WR_TIMER_THREAD, WrTimer, TIMER) TG_DEF(SvrStartupHandler, SvrStartupHandler, QUEUE_THREAD, ThreadCountPair(observer::ObServerStartupTaskHandler::get_thread_num(), observer::ObServerStartupTaskHandler::get_thread_num()), observer::ObServerStartupTaskHandler::MAX_QUEUED_TASK_NUM) +TG_DEF(TenantTTLManager, TTLManager, TIMER) +TG_DEF(TenantTabletTTLMgr, TTLTabletMgr, TIMER) TG_DEF(TntSharedTimer, TntSharedTimer, TIMER) #endif diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index ea168615a7..157dd1e4a9 100755 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1606,7 +1606,19 @@ ERRSIM_DEF_STR_LIST(errsim_module_types, OB_TENANT_PARAMETER, "", ERRSIM_DEF_DBL(errsim_module_error_percentage, OB_TENANT_PARAMETER, "0", "[0,100]", "the percentage of the error happened to errsim module. " "Range: [0, 100] in percentage", - ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)) + +// ttl +DEF_STR_WITH_CHECKER(kv_ttl_duty_duration, OB_TENANT_PARAMETER, "", common::ObTTLDutyDurationChecker, + "ttl background task working time duration" + "begin_time or end_time in Range, e.g., [23:00:00, 24:00:00]", + ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_TIME(kv_ttl_history_recycle_interval, OB_TENANT_PARAMETER, "7d", "[1d, 180d]", + "the time to recycle ttl history. Range: [1d, 180d]", + ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_BOOL(enable_kv_ttl, OB_TENANT_PARAMETER, "False", + "specifies whether ttl task is enbled", + ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_STR_WITH_CHECKER(sql_protocol_min_tls_version, OB_CLUSTER_PARAMETER, "none", common::ObConfigSQLTlsVersionChecker, "SQL SSL control options, used to specify the minimum SSL/TLS version number. " diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 300db4b49c..6cd377a82b 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -105,7 +105,10 @@ namespace transaction { namespace concurrency_control { class ObMultiVersionGarbageCollector; // MVCC GC } - +namespace table +{ + class ObHTableLockMgr; +} namespace logservice { class ObLogService; @@ -177,6 +180,11 @@ namespace storage { class MockTenantModuleEnv; } +namespace table +{ + class ObTTLService; +} + namespace share { class ObCgroupCtrl; @@ -308,7 +316,9 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPool 0 + && NULL != table_schema.get_ttl_definition().ptr()) { + const ObString ttl_definition = table_schema.get_ttl_definition(); + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "TTL = (%.*s) ", + ttl_definition.length(), ttl_definition.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ttl definition", K(ret), K(ttl_definition)); + } + } + if (OB_SUCC(ret) && !strict_compat_ && !is_index_tbl + && table_schema.get_kv_attributes().length() > 0 + && NULL != table_schema.get_kv_attributes().ptr()) { + const ObString kv_attributes = table_schema.get_kv_attributes(); + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "KV_ATTRIBUTES = (%.*s) ", + kv_attributes.length(), kv_attributes.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print kv attributes", K(ret), K(kv_attributes)); + } + } if (OB_SUCC(ret) && pos > 0) { pos -= 1; buf[pos] = '\0'; // remove trailer space @@ -2236,6 +2254,26 @@ int ObSchemaPrinter::print_table_definition_table_options( OB_LOG(WARN, "fail to print progressive merge num", K(ret), K(table_schema)); } } + if (OB_SUCC(ret) && !is_index_tbl && !strict_compat_) { + const ObString ttl_definition = table_schema.get_ttl_definition(); + if (ttl_definition.empty()) { + // do nothing + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "TTL = (%.*s) ", + ttl_definition.length(), ttl_definition.ptr()))) { + OB_LOG(WARN, "fail to print ttl definition", K(ret), K(ttl_definition)); + } + } + + if (OB_SUCC(ret) && !is_index_tbl && !strict_compat_) { + const ObString kv_attributes = table_schema.get_kv_attributes(); + if (kv_attributes.empty()) { + // do nothing + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "KV_ATTRIBUTES = (%.*s) ", + kv_attributes.length(), kv_attributes.ptr()))) { + OB_LOG(WARN, "fail to print kv attributes", K(ret), K(kv_attributes)); + } + } + return ret; } diff --git a/src/share/schema/ob_schema_retrieve_utils.ipp b/src/share/schema/ob_schema_retrieve_utils.ipp index b99d836ff4..e6db8793c4 100644 --- a/src/share/schema/ob_schema_retrieve_utils.ipp +++ b/src/share/schema/ob_schema_retrieve_utils.ipp @@ -1316,6 +1316,10 @@ int ObSchemaRetrieveUtils::fill_table_schema( result, external_file_format, table_schema, true/*skip null*/, true/*ignore column error*/, external_file_format); EXTRACT_VARCHAR_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE( result, external_file_pattern, table_schema, true/*skip null*/, true/*ignore column error*/, external_file_pattern); + EXTRACT_VARCHAR_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE( + result, ttl_definition, table_schema, true, ignore_column_error, ""); + EXTRACT_VARCHAR_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE( + result, kv_attributes, table_schema, true, ignore_column_error, ""); } if (OB_SUCC(ret) && OB_FAIL(fill_sys_table_lob_tid(table_schema))) { SHARE_SCHEMA_LOG(WARN, "fail to fill lob table id for inner table", K(ret), K(table_schema.get_table_id())); diff --git a/src/share/schema/ob_schema_service.cpp b/src/share/schema/ob_schema_service.cpp index 720e1e67c1..81028b29f6 100644 --- a/src/share/schema/ob_schema_service.cpp +++ b/src/share/schema/ob_schema_service.cpp @@ -516,6 +516,12 @@ int AlterTableSchema::assign(const ObTableSchema &src_schema) } } } + if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(src_schema.ttl_definition_, ttl_definition_))) { + LOG_WARN("Fail to deep copy ttl definition string", K(ret)); + } + if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(src_schema.kv_attributes_, kv_attributes_))) { + LOG_WARN("Fail to deep copy ttl definition string", K(ret)); + } return ret; } diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 04f2a74a54..8e94d9c283 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -1650,6 +1650,15 @@ int ObTableSchema::assign(const ObTableSchema &src_schema) } } + + if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(src_schema.ttl_definition_, ttl_definition_))) { + LOG_WARN("deep copy ttl definition failed", K(ret)); + } + + if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(src_schema.kv_attributes_, kv_attributes_))) { + LOG_WARN("deep copy kv attributes failed", K(ret)); + } + if (OB_FAIL(ret)) { LOG_WARN("failed to assign table schema", K(ret)); } @@ -3113,6 +3122,8 @@ int64_t ObTableSchema::get_convert_size() const convert_size += external_file_location_.length() + 1; convert_size += external_file_location_access_info_.length() + 1; convert_size += external_file_pattern_.length() + 1; + convert_size += ttl_definition_.length() + 1; + convert_size += kv_attributes_.length() + 1; return convert_size; } @@ -3197,6 +3208,8 @@ void ObTableSchema::reset() external_file_location_.reset(); external_file_location_access_info_.reset(); external_file_pattern_.reset(); + ttl_definition_.reset(); + kv_attributes_.reset(); ObSimpleTableSchemaV2::reset(); } @@ -6192,6 +6205,14 @@ OB_DEF_SERIALIZE(ObTableSchema) external_file_pattern_); } }(); + + if (OB_SUCC(ret)) { + OB_UNIS_ENCODE(ttl_definition_); + } + + if (OB_SUCC(ret)) { + OB_UNIS_ENCODE(kv_attributes_); + } return ret; } @@ -6303,6 +6324,8 @@ OB_DEF_DESERIALIZE(ObTableSchema) ObString create_host; ObString table_name; ObString expire_info; + ObString ttl_definition; + ObString kv_attributes; LST_DO_CODE(OB_UNIS_DECODE, tenant_id_, @@ -6541,6 +6564,20 @@ OB_DEF_DESERIALIZE(ObTableSchema) } } }(); + + if (OB_SUCC(ret)) { + OB_UNIS_DECODE(ttl_definition); + if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(ttl_definition, ttl_definition_))) { + LOG_WARN("deep_copy_str failed", K(ret), K(ttl_definition)); + } + } + + if (OB_SUCC(ret)) { + OB_UNIS_DECODE(kv_attributes); + if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(kv_attributes, kv_attributes_))) { + LOG_WARN("deep_copy_str failed", K(ret), K(kv_attributes)); + } + } return ret; } @@ -6678,6 +6715,8 @@ OB_DEF_SERIALIZE_SIZE(ObTableSchema) OB_UNIS_ADD_LEN(external_file_location_access_info_); OB_UNIS_ADD_LEN(external_file_format_); OB_UNIS_ADD_LEN(external_file_pattern_); + OB_UNIS_ADD_LEN(ttl_definition_); + OB_UNIS_ADD_LEN(kv_attributes_); return len; } diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 44427991a1..8fa6fbf2e7 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1029,6 +1029,8 @@ public: // Copy all constraint information in src_schema int assign_constraint(const ObTableSchema &other); void clear_constraint(); + int set_ttl_definition(const common::ObString &ttl_definition) { return deep_copy_str(ttl_definition, ttl_definition_); } + int set_kv_attributes(const common::ObString &kv_attributes) { return deep_copy_str(kv_attributes, kv_attributes_); } //get methods bool is_valid() const; @@ -1115,6 +1117,8 @@ public: inline const common::ObString &get_expire_info() const { return expire_info_; } inline ObViewSchema &get_view_schema() { return view_schema_; } inline const ObViewSchema &get_view_schema() const { return view_schema_; } + inline const common::ObString &get_ttl_definition() const { return ttl_definition_; } + inline const common::ObString &get_kv_attributes() const { return kv_attributes_; } bool has_check_constraint() const; inline bool has_constraint() const { return cst_cnt_ > 0; } bool is_column_in_check_constraint(const uint64_t col_id) const; @@ -1631,6 +1635,12 @@ protected: common::ObString external_file_location_; common::ObString external_file_location_access_info_; common::ObString external_file_pattern_; + + // table ttl + common::ObString ttl_definition_; + + // kv attributes + common::ObString kv_attributes_; }; class ObPrintableTableSchema final : public ObTableSchema diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index 51c34c3c4e..af70559499 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -2666,6 +2666,14 @@ int ObTableSqlService::gen_table_dml( ret = OB_NOT_SUPPORTED; LOG_WARN("external table is not support before 4.2", K(ret), K(table)); } else { + if (data_version < DATA_VERSION_4_2_1_0 + && (!table.get_ttl_definition().empty() || !table.get_kv_attributes().empty())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl definition and kv attributes is not supported in version less than 4.2.1", + "ttl_definition", table.get_ttl_definition().empty(), + "kv_attributes", table.get_kv_attributes().empty()); + } else {} + if (OB_SUCC(ret)) { const ObPartitionOption &part_option = table.get_part_option(); const ObPartitionOption &sub_part_option = table.get_sub_part_option(); const char *expire_info = table.get_expire_info().length() <= 0 ? @@ -2681,6 +2689,10 @@ int ObTableSqlService::gen_table_dml( const int64_t sub_part_num = PARTITION_LEVEL_TWO == table.get_part_level() && table.has_sub_part_template_def() ? sub_part_option.get_part_num() : 0; + const char *ttl_definition = table.get_ttl_definition().empty() ? + "" : table.get_ttl_definition().ptr(); + const char *kv_attributes = table.get_kv_attributes().empty() ? + "" : table.get_kv_attributes().ptr(); if (OB_FAIL(check_table_options(table))) { LOG_WARN("fail to check table option", K(ret), K(table)); } else if (data_version < DATA_VERSION_4_1_0_0 && 0 != table.get_table_flags()) { @@ -2782,10 +2794,15 @@ int ObTableSqlService::gen_table_dml( && OB_FAIL(dml.add_column("external_file_format", ObHexEscapeSqlStr(table.get_external_file_format())))) || (data_version >= DATA_VERSION_4_2_0_0 && OB_FAIL(dml.add_column("external_file_pattern", ObHexEscapeSqlStr(table.get_external_file_pattern())))) + || (data_version >= DATA_VERSION_4_2_1_0 + && OB_FAIL(dml.add_column("ttl_definition", ObHexEscapeSqlStr(ttl_definition)))) + || (data_version >= DATA_VERSION_4_2_1_0 + && OB_FAIL(dml.add_column("kv_attributes", ObHexEscapeSqlStr(kv_attributes)))) ) { LOG_WARN("add column failed", K(ret)); } } + } return ret; } @@ -2801,7 +2818,14 @@ int ObTableSqlService::gen_table_options_dml( LOG_WARN("check ddl allowd failed", K(ret), K(table)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(table.get_tenant_id(), data_version))) { LOG_WARN("failed to get data version", K(ret)); - } else { + } else if (data_version < DATA_VERSION_4_2_1_0 + && (!table.get_ttl_definition().empty() || !table.get_kv_attributes().empty())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl definition and kv attributes is not supported in version less than 4.2.1", + "ttl_definition", table.get_ttl_definition().empty(), + "kv_attributes", table.get_kv_attributes().empty()); + } else {} + if (OB_SUCC(ret)) { const ObPartitionOption &part_option = table.get_part_option(); const ObPartitionOption &sub_part_option = table.get_sub_part_option(); const char *table_name = table.get_table_name_str().length() <= 0 ? @@ -2817,6 +2841,10 @@ int ObTableSqlService::gen_table_options_dml( const char *encryption = table.get_encryption_str().empty() ? "" : table.get_encryption_str().ptr(); const int64_t INVALID_REPLICA_NUM = -1; + const char *ttl_definition = table.get_ttl_definition().length() <= 0 ? + "" : table.get_ttl_definition().ptr(); + const char *kv_attributes = table.get_kv_attributes().length() <= 0 ? + "" : table.get_kv_attributes().ptr(); if (OB_FAIL(check_table_options(table))) { LOG_WARN("fail to check table option", K(ret), K(table)); @@ -2887,6 +2915,10 @@ int ObTableSqlService::gen_table_options_dml( || (OB_FAIL(dml.add_column("tablet_id", table.get_tablet_id().id()))) || ((data_version >= DATA_VERSION_4_1_0_0 || update_object_status_ignore_version) && OB_FAIL(dml.add_column("object_status", static_cast (table.get_object_status())))) + || (data_version >= DATA_VERSION_4_2_1_0 + && OB_FAIL(dml.add_column("ttl_definition", ObHexEscapeSqlStr(ttl_definition)))) + || (data_version >= DATA_VERSION_4_2_1_0 + && OB_FAIL(dml.add_column("kv_attributes", ObHexEscapeSqlStr(kv_attributes)))) ) { LOG_WARN("add column failed", K(ret)); } diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index ead0d94050..11fca1385d 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -25,6 +25,8 @@ #include "common/ob_range.h" #include "rpc/obrpc/ob_poc_rpc_server.h" +#include "share/table/ob_table_ttl_common.h" +#include "common/rowkey/ob_rowkey.h" namespace oceanbase { namespace common @@ -236,6 +238,7 @@ struct ObTableOperationType INCREMENT = 6, APPEND = 7, SCAN = 8, + TTL = 9, // internal type for ttl executor cache key INVALID = 15 }; }; @@ -330,6 +333,33 @@ private: ObTableOperationType::Type operation_type_; }; +class ObTableTTLOperation +{ +public: + ObTableTTLOperation(uint64_t tenant_id, uint64_t table_id, const ObTTLTaskParam ¶, + uint64_t del_row_limit, ObRowkey start_rowkey) + : tenant_id_(tenant_id), table_id_(table_id), max_version_(para.max_version_), + time_to_live_(para.ttl_), is_htable_(para.is_htable_), del_row_limit_(del_row_limit), + start_rowkey_(start_rowkey) + {} + + ~ObTableTTLOperation() {} + bool is_valid() const + { + return common::OB_INVALID_TENANT_ID != tenant_id_ && common::OB_INVALID_ID != table_id_ && + (!is_htable_ || max_version_ > 0 || time_to_live_ > 0) && del_row_limit_ > 0; + } + TO_STRING_KV(K_(tenant_id), K_(table_id), K_(max_version), K_(time_to_live), K_(is_htable), K_(del_row_limit), K_(start_rowkey)); +public: + uint64_t tenant_id_; + uint64_t table_id_; + int32_t max_version_; + int32_t time_to_live_; + bool is_htable_; + uint64_t del_row_limit_; + ObRowkey start_rowkey_; +}; + /// common result for ObTable class ObTableResult { @@ -904,6 +934,30 @@ enum class ObTableDirectLoadOperationType { MAX_TYPE }; +class ObTableTTLOperationResult +{ +public: + ObTableTTLOperationResult() + : ttl_del_rows_(0), + max_version_del_rows_(0), + scan_rows_(0), + end_rowkey_() + {} + ~ObTableTTLOperationResult() {} + uint64_t get_ttl_del_row() { return ttl_del_rows_; } + uint64_t get_max_version_del_row() { return max_version_del_rows_; } + uint64_t get_del_row() { return ttl_del_rows_ + max_version_del_rows_; } + uint64_t get_scan_row() { return scan_rows_; } + common::ObString get_end_rowkey() { return end_rowkey_; } + TO_STRING_KV(K_(ttl_del_rows), K_(max_version_del_rows), K_(scan_rows), K_(end_rowkey)); +public: + uint64_t ttl_del_rows_; + uint64_t max_version_del_rows_; + uint64_t scan_rows_; + common::ObString end_rowkey_; +}; + + } // end namespace table } // end namespace oceanbase diff --git a/src/share/table/ob_table_ttl_common.h b/src/share/table/ob_table_ttl_common.h new file mode 100644 index 0000000000..2506f0db43 --- /dev/null +++ b/src/share/table/ob_table_ttl_common.h @@ -0,0 +1,116 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SHARE_TABLE_OB_TABLE_TTL_COMMON_ +#define OCEANBASE_SHARE_TABLE_OB_TABLE_TTL_COMMON_ + +#include "share/ob_ls_id.h" +#include "common/ob_tablet_id.h" +#include "share/rc/ob_tenant_base.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTTLTaskParam final +{ +public: + ObTTLTaskParam() + : ttl_(0), + max_version_(0), + is_htable_(false), + tenant_id_(OB_INVALID_ID), + user_id_(OB_INVALID_ID), + database_id_(OB_INVALID_ID), + table_id_(OB_INVALID_ID) + {} + + bool is_valid() const + { + return tenant_id_ != OB_INVALID_ID && + user_id_ != OB_INVALID_ID && + database_id_ != OB_INVALID_ID && + table_id_ != OB_INVALID_ID; + } + + bool operator==(const ObTTLTaskParam& param) const + { + return ttl_ == param.ttl_ && + max_version_ == param.max_version_ && + is_htable_ == param.is_htable_ && + tenant_id_ == param.tenant_id_ && + database_id_ == param.database_id_ && + user_id_ == param.user_id_ && + table_id_ == param.table_id_; + } + + TO_STRING_KV(K_(ttl), K_(max_version), K_(is_htable), K_(tenant_id), + K_(user_id), K_(database_id), K_(table_id)); +public: + int32_t ttl_; + int32_t max_version_; + bool is_htable_; + int64_t tenant_id_; + int64_t user_id_; + int64_t database_id_; + uint64_t table_id_; +}; + +class ObTTLTaskInfo final +{ +public: + ObTTLTaskInfo() + : task_id_(OB_INVALID_ID), + tablet_id_(), + table_id_(OB_INVALID_ID), + is_user_trigger_(true), + row_key_(), + ttl_del_cnt_(), + max_version_del_cnt_(0), + scan_cnt_(0), + err_code_(OB_SUCCESS), + tenant_id_(common::OB_INVALID_TENANT_ID), + ls_id_(), + consumer_group_id_(0) + { + } + + bool is_valid() const + { + return common::OB_INVALID_ID != task_id_; + } + const common::ObTabletID &get_tablet_id() const { return tablet_id_; } + + TO_STRING_KV(K_(task_id), K_(tablet_id), K_(table_id), K_(is_user_trigger), + K_(is_user_trigger), K_(row_key), K_(ttl_del_cnt), + K_(max_version_del_cnt), K_(scan_cnt), K_(err_code), + K_(tenant_id), K_(ls_id), K_(consumer_group_id)); + + int64_t task_id_; + common::ObTabletID tablet_id_; + uint64_t table_id_; + bool is_user_trigger_; + common::ObString row_key_; + int64_t ttl_del_cnt_; + int64_t max_version_del_cnt_; + int64_t scan_cnt_; + int64_t err_code_; + int64_t tenant_id_; + share::ObLSID ls_id_; + int64_t consumer_group_id_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_SHARE_TABLE_OB_TABLE_TTL_COMMON_ */ \ No newline at end of file diff --git a/src/share/table/ob_ttl_util.cpp b/src/share/table/ob_ttl_util.cpp new file mode 100644 index 0000000000..d90c0e9335 --- /dev/null +++ b/src/share/table/ob_ttl_util.cpp @@ -0,0 +1,1139 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER + +#include "share/table/ob_ttl_util.h" +#include "share/ob_max_id_fetcher.h" +#include "share/ob_srv_rpc_proxy.h" +#include "share/ob_server_status.h" +#include "share/schema/ob_schema_utils.h" +#include "rootserver/ob_root_service.h" +#include "observer/omt/ob_tenant_timezone_mgr.h" +#include "share/schema/ob_multi_version_schema_service.h" +#include "lib/stat/ob_diagnose_info.h" +#include "share/location_cache/ob_location_service.h" + +using namespace oceanbase::share; + +namespace oceanbase +{ +namespace common +{ + +bool ObTTLTime::is_same_day(int64_t ttl_time1, int64_t ttl_time2) +{ + time_t param1 = static_cast(ttl_time1 / 1000000l); + time_t param2 = static_cast(ttl_time2 / 1000000l); + + struct tm tm1, tm2; + ::localtime_r(¶m1, &tm1); + ::localtime_r(¶m2, &tm2); + + return (tm1.tm_yday == tm2.tm_yday); +} + +bool ObTTLUtil::extract_val(const char* ptr, uint64_t len, int& val) +{ + char buffer[16] = {0}; + bool bool_ret = false; + for (int i = 0; i < len; ++i) { + if (ptr[i] == ' ') { + continue; + } else if (ptr[i] >= '0' && ptr[i] <= '9') { + bool_ret = true; + MEMCPY(buffer, ptr + i, len - i > 2 ? len - i : 2); + break; + } + } + val = atoi(buffer); + return bool_ret; +} + +int ObTTLUtil::parse_ttl_daytime(ObString& in, ObTTLDayTime& daytime) +{ + int ret = OB_SUCCESS; + + const char* first_split = in.find(':'); + const char* second_split = in.reverse_find(':'); + + if (in.contains(first_split) && + in.contains(second_split) && + first_split < second_split) { + if (extract_val(in.ptr(), first_split - in.ptr(), daytime.hour_) && + extract_val(first_split + 1, second_split - first_split - 1, daytime.min_) && + extract_val(second_split + 1, in.length() + in.ptr() - second_split, daytime.sec_)) { + } else { + ret = OB_INVALID_CONFIG; + LOG_WARN("illegal input string", K(ret), K(in)); + } + } else { + ret = OB_INVALID_CONFIG; + LOG_WARN("illegal input string", K(ret), K(in)); + } + + return ret; +} + +int ObTTLUtil::parse(const char* str, ObTTLDutyDuration& duration) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(str) || strlen(str) == 0) { + duration.not_set_ = true; + } else { + ObString in_str(str); + const char* begin = in_str.find('['); + const char* split = in_str.find(','); + const char* end = in_str.reverse_find(']'); + + if (OB_ISNULL(begin) || OB_ISNULL(split) || OB_ISNULL(end)) { + ret = OB_INVALID_CONFIG; + LOG_WARN("fail to parse str", K(ret)); + } else { + ObString first_param, second_param; + first_param.assign_ptr(begin + 1, split - begin - 1); + second_param.assign_ptr(split + 1, end - split - 1); + + if (OB_FAIL(parse_ttl_daytime(first_param, duration.begin_)) || + OB_FAIL(parse_ttl_daytime(second_param, duration.end_))) { + LOG_WARN("fail to parse daytime", K(ret)); + } else { + duration.not_set_ = false; + } + } + } + + return ret; +} + +bool ObTTLUtil::current_in_duration(ObTTLDutyDuration& duration) +{ + bool bret = false; + if (!duration.not_set_) { + time_t now; + time(&now); + struct tm *t = localtime(&now); + uint32_t begin = duration.begin_.sec_ + 60 * (duration.begin_.min_ + 60 * duration.begin_.hour_); + uint32_t end = duration.end_.sec_ + 60 * (duration.end_.min_ + 60 * duration.end_.hour_); + uint32_t current = t->tm_sec + 60 * (t->tm_min + 60 * t->tm_hour); + bret = (begin <= current) & ( current <= end); + } + return bret; +} + +int ObTTLUtil::insert_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatus& task) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t affect_rows = 0; + + if (OB_FAIL(sql.assign_fmt("INSERT INTO %s " + "(gmt_create, gmt_modified, tenant_id, table_id, tablet_id, " + "task_id, task_start_time, task_update_time, trigger_type, status," + " ttl_del_cnt, max_version_del_cnt, scan_cnt, ret_code, row_key)" + " VALUE " + "(now(), now(), %ld, %ld, %ld," + " %ld, %ld, %ld, %ld, %ld, " + " %ld, %ld, %ld,'%.*s', ", + tname, + tenant_id, task.table_id_, task.tablet_id_, + task.task_id_, task.task_start_time_, task.task_update_time_, task.trigger_type_, + task.status_, task.ttl_del_cnt_, task.max_version_del_cnt_, + task.scan_cnt_, task.ret_code_.length(), task.ret_code_.ptr()))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(sql_append_hex_escape_str(task.row_key_, sql))) { + LOG_WARN("fail to append rowkey", K(ret)); + } else if (OB_FAIL(sql.append(")"))) { + LOG_WARN("fail to append"); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affect_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql)); + } else if (affect_rows != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_INFO("execute sql, affect rows != 1", K(ret), K(sql)); + } else { + LOG_INFO("success to execute sql", K(ret), K(sql)); + } + + return ret; +} + +int ObTTLUtil::update_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatusKey& key, + ObTTLStatusFieldArray& update_fields) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + + if (OB_FAIL(sql.assign_fmt("UPDATE %s SET ", tname))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } + + // FILED_NAME = value string construct + for (size_t i = 0; OB_SUCC(ret) && i < update_fields.count(); ++i) { + ObTTLStatusField& field = update_fields.at(i); + + if (OB_FAIL(sql.append_fmt("%s =", field.field_name_.ptr()))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (field.type_ == ObTTLStatusField::INT_TYPE) { + if (OB_FAIL(sql.append_fmt("%ld", field.data_.int_))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } else if (field.type_ == ObTTLStatusField::UINT_TYPE) { + if (OB_FAIL(sql.append_fmt("%ld", field.data_.uint_))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } else if (field.type_ == ObTTLStatusField::STRING_TYPE) { + if (OB_FAIL(sql.append_fmt("%s", field.data_.str_.ptr()))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql append fmt failed", K(ret)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(sql.append_fmt("%s", i == update_fields.count() - 1 ? " " : ","))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } + } + + // WHERE FILTER + if (OB_FAIL(ret)) { + } else if (OB_FAIL(sql.append_fmt(" WHERE " + "tenant_id = %ld AND table_id = %ld AND tablet_id = %ld AND task_id = %ld", + key.tenant_id_, key.table_id_, key.tablet_id_, key.task_id_))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + + int64_t affect_rows = 0; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affect_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql)); + } else if (affect_rows != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_INFO("execute sql, affect rows != 1", K(ret), K(sql)); + } else { + LOG_INFO("success to execute sql", K(ret), K(sql)); + } + + return ret; +} + +int ObTTLUtil::update_ttl_task_all_fields(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatus& task) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t affect_rows = 0; + + if (OB_FAIL(sql.assign_fmt("UPDATE %s SET " + "task_start_time = %ld, task_update_time = %ld, trigger_type = %ld, status = %ld," + " ttl_del_cnt = %ld, max_version_del_cnt = %ld, scan_cnt = %ld, ret_code = '%.*s'," + " row_key = ", + tname, task.task_start_time_, task.task_update_time_, task.trigger_type_, task.status_, + task.ttl_del_cnt_, task.max_version_del_cnt_, task.scan_cnt_, task.ret_code_.length(), + task.ret_code_.ptr()))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(sql_append_hex_escape_str(task.row_key_, sql))) { + LOG_WARN("fail to append rowkey", K(ret)); + } else if (OB_FAIL(sql.append_fmt(" WHERE tenant_id = %ld AND table_id = %ld" + " AND tablet_id = %ld AND task_id = %ld ", + tenant_id, task.table_id_, task.tablet_id_, task.task_id_))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affect_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql)); + } else { + LOG_INFO("success to execute sql", K(ret), K(sql)); + } + + return ret; +} + +int ObTTLUtil::delete_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatusKey& key, + int64_t &affect_rows) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + + if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE " + "tenant_id = %ld AND table_id = %ld " + "AND tablet_id = %ld AND task_id = %ld", + tname, + tenant_id, key.table_id_, + key.tablet_id_, key.task_id_))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affect_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql)); + } else { + LOG_INFO("success to execute sql", K(ret), K(sql)); + } + + return ret; +} + +int ObTTLUtil::read_ttl_tasks(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatusFieldArray& filters, + ObTTLStatusArray& result_arr, + bool for_update /*false*/, + common::ObIAllocator *allocator /*NULL*/) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + + if (OB_FAIL(sql.assign_fmt("SELECT * FROM %s where ", tname))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } + + // FILED_NAME = value string construct + for (size_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) { + ObTTLStatusField& field = filters.at(i); + + if (OB_FAIL(sql.append_fmt("%s = ", field.field_name_.ptr()))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (field.type_ == ObTTLStatusField::INT_TYPE) { + if (OB_FAIL(sql.append_fmt("%ld", field.data_.int_))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } else if (field.type_ == ObTTLStatusField::UINT_TYPE) { + if (OB_FAIL(sql.append_fmt("%ld", field.data_.uint_))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } else if (field.type_ == ObTTLStatusField::STRING_TYPE) { + if (OB_FAIL(sql.append_fmt("%s", field.data_.str_.ptr()))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql append fmt failed", K(ret)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(sql.append_fmt("%s", i == filters.count() - 1 ? "" : " AND "))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } + } + + if (OB_SUCC(ret) && for_update) { + if (OB_FAIL(sql.append_fmt(" for update"))) { + LOG_WARN("sql append fmt failed", K(ret)); + } + } + + + if (OB_SUCC(ret)) { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult* result = nullptr; + if (OB_FAIL(proxy.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + LOG_WARN("fail to execute sql", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected, query result must not be NULL", K(ret)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("fail to get next row", K(ret)); + } + } else { + size_t idx = result_arr.count(); + ObTTLStatus task; + if (OB_FAIL(result_arr.push_back(task))) { + LOG_WARN("fail to push back task", K(ret), K(result_arr.count())); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "tenant_id", result_arr.at(idx).tenant_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "table_id", result_arr.at(idx).table_id_, uint64_t); + + EXTRACT_INT_FIELD_MYSQL(*result, "tablet_id", result_arr.at(idx).tablet_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "task_id", result_arr.at(idx).task_id_, uint64_t); + + EXTRACT_INT_FIELD_MYSQL(*result, "task_start_time", result_arr.at(idx).task_start_time_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "task_update_time", result_arr.at(idx).task_update_time_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "trigger_type", result_arr.at(idx).trigger_type_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "status", result_arr.at(idx).status_, int64_t); + + EXTRACT_INT_FIELD_MYSQL(*result, "ttl_del_cnt", result_arr.at(idx).ttl_del_cnt_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "max_version_del_cnt", result_arr.at(idx).max_version_del_cnt_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "scan_cnt", result_arr.at(idx).scan_cnt_, uint64_t); + if (OB_SUCC(ret) && OB_NOT_NULL(allocator)) { + ObString rowkey; + char *rowkey_buf = nullptr; + EXTRACT_VARCHAR_FIELD_MYSQL(*result, "row_key", rowkey); + if (OB_SUCC(ret) && !rowkey.empty()) { + if (OB_ISNULL(rowkey_buf = static_cast(allocator->alloc(rowkey.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failt to allocate memory", K(ret), K(rowkey)); + } else { + MEMCPY(rowkey_buf, rowkey.ptr(), rowkey.length()); + result_arr.at(idx).row_key_.assign(rowkey_buf, rowkey.length()); + } + } + } + + if (OB_SUCC(ret) && OB_NOT_NULL(allocator)) { + ObString err_msg; + char *err_buf = nullptr; + EXTRACT_VARCHAR_FIELD_MYSQL(*result, "ret_code", err_msg); + if (OB_SUCC(ret) && !err_msg.empty()) { + if (OB_ISNULL(err_buf = static_cast(allocator->alloc(err_msg.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failt to allocate memory", K(ret), K(err_msg)); + } else { + MEMCPY(err_buf, err_msg.ptr(), err_msg.length()); + result_arr.at(idx).ret_code_.assign(err_buf, err_msg.length()); + } + } + } + } + } + } + } + } + } + + return ret; +} + +int ObTTLUtil::read_tenant_ttl_task(uint64_t tenant_id, + common::ObISQLClient& sql_client, + ObTTLStatus& ttl_record, + ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + if (!is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("SELECT * FROM %s where table_id = '%ld'", OB_ALL_KV_TTL_TASK_TNAME, TTL_TENNAT_TASK_TABLE_ID))) { + LOG_WARN("fail to append sql", KR(ret), K(tenant_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *result = nullptr; + if (OB_FAIL(sql_client.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + LOG_WARN("fail to execute sql", KR(ret), K(tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get sql result", KR(ret), K(tenant_id), K(sql)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next result"); + } + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "tenant_id", ttl_record.tenant_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "table_id", ttl_record.table_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "tablet_id", ttl_record.tablet_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "task_id", ttl_record.task_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "task_start_time", ttl_record.task_start_time_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "task_update_time", ttl_record.task_update_time_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "trigger_type", ttl_record.trigger_type_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "status", ttl_record.status_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "ttl_del_cnt", ttl_record.ttl_del_cnt_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "max_version_del_cnt", ttl_record.max_version_del_cnt_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "scan_cnt", ttl_record.scan_cnt_, uint64_t); + if (OB_SUCC(ret) && OB_NOT_NULL(allocator)) { + ObString rowkey; + char *rowkey_buf = nullptr; + EXTRACT_VARCHAR_FIELD_MYSQL(*result, "row_key", rowkey); + if (OB_SUCC(ret) && !rowkey.empty()) { + if (OB_ISNULL(rowkey_buf = static_cast(allocator->alloc(rowkey.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failt to allocate memory", K(ret), K(rowkey)); + } else { + MEMCPY(rowkey_buf, rowkey.ptr(), rowkey.length()); + ttl_record.row_key_.assign(rowkey_buf, rowkey.length()); + } + } + } + if (OB_SUCC(ret) && OB_NOT_NULL(allocator)) { + ObString err_msg; + char *err_buf = nullptr; + EXTRACT_VARCHAR_FIELD_MYSQL(*result, "ret_code", err_msg); + if (OB_SUCC(ret) && !err_msg.empty()) { + if (OB_ISNULL(err_buf = static_cast(allocator->alloc(err_msg.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failt to allocate memory", K(ret), K(err_msg)); + } else { + MEMCPY(err_buf, err_msg.ptr(), err_msg.length()); + ttl_record.ret_code_.assign(err_buf, err_msg.length()); + } + } + } + } + } + } + return ret; +} + +bool ObTTLUtil::check_can_do_work() { + bool bret = true; + int ret = OB_SUCCESS; + int64_t tenant_id = MTL_ID(); + uint64_t tenant_data_version = 0;; + if (GCTX.is_standby_cluster()) { + bret = false; + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + bret = false; + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_0) { + bret = false; + LOG_DEBUG("TTL can not work with data version less than 4_2_1", K(tenant_data_version)); + } else if (is_user_tenant(tenant_id)) { + if (OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id), tenant_data_version))) { + bret = false; + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_0) { + bret = false; + LOG_DEBUG("TTL can not work with data version less than 4_2_1", K(tenant_data_version)); + } + } + return bret; +} + + +bool ObTTLUtil::check_can_process_tenant_tasks(uint64_t tenant_id) +{ + bool bret = false; + + if (OB_INVALID_TENANT_ID == tenant_id) { + LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid tenant id"); + } else { + int ret = OB_SUCCESS; + bool is_restore = true; + if (OB_FAIL(share::schema::ObMultiVersionSchemaService::get_instance(). + check_tenant_is_restore(NULL, tenant_id, is_restore))) { + if (OB_TENANT_NOT_EXIST != ret) { + LOG_WARN("fail to check tenant is restore", KR(ret), K(tenant_id), K(common::lbt())); + } else { + ret = OB_SUCCESS; + } + } else { + bret = !is_restore; + } + } + return bret; +} + +int ObTTLUtil::move_task_to_history_table(uint64_t tenant_id, uint64_t task_id, + common::ObMySQLTransaction& proxy, + int64_t batch_size, int64_t &move_rows) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t insert_rows = 0; + int64_t delete_rows = 0; + if (OB_FAIL(sql.assign_fmt("replace into %s select * from %s " + " where task_id = %ld and tablet_id != -1 and table_id != -1" + " order by tenant_id, task_id, table_id, tablet_id LIMIT %ld", + share::OB_ALL_KV_TTL_TASK_HISTORY_TNAME, + share::OB_ALL_KV_TTL_TASK_TNAME, + task_id, batch_size))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), insert_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("delete from %s" + " where task_id = %ld and tablet_id != -1 and table_id != -1" + " order by tenant_id, task_id, table_id, tablet_id LIMIT %ld ", + share::OB_ALL_KV_TTL_TASK_TNAME, + task_id, batch_size))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), delete_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(tenant_id)); + } else { + move_rows = delete_rows; + LOG_INFO("success to execute sql", K(ret), K(tenant_id), K(sql), K(insert_rows), K(delete_rows)); + } + + return ret; +} + +// only one record left in this situation +int ObTTLUtil::move_tenant_task_to_history_table(uint64_t tenant_id, uint64_t task_id, + common::ObMySQLTransaction& proxy) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t insert_rows = 0; + int64_t delete_rows = 0; + if (OB_FAIL(sql.assign_fmt("insert into %s select * from %s " + " where task_id = %ld and tablet_id = %ld", + share::OB_ALL_KV_TTL_TASK_HISTORY_TNAME, + share::OB_ALL_KV_TTL_TASK_TNAME, + task_id, TTL_TENNAT_TASK_TABLET_ID))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), insert_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("delete from %s" + " where task_id = %ld and tablet_id = %ld", + share::OB_ALL_KV_TTL_TASK_TNAME, + task_id, TTL_TENNAT_TASK_TABLET_ID))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), delete_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(tenant_id)); + } else { + LOG_INFO("success to execute sql", K(ret), K(tenant_id), K(sql), K(insert_rows), K(delete_rows)); + } + + return ret; + +} + +int ObTTLUtil::replace_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatus& task) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t affect_rows = 0; + + if (OB_FAIL(sql.assign_fmt("REPLACE INTO %s " + "(gmt_create, gmt_modified, tenant_id, table_id, tablet_id, " + "task_id, task_start_time, task_update_time, trigger_type, status," + " ttl_del_cnt, max_version_del_cnt, scan_cnt, ret_code, row_key)" + " VALUE " + "(now(), now(), %ld, %ld, %ld," + " %ld, %ld, %ld, %ld, %ld, " + " %ld, %ld, %ld,'%.*s', ", + tname, // 0 + tenant_id, task.table_id_, task.tablet_id_, + task.task_id_, task.task_start_time_, task.task_update_time_, task.trigger_type_, task.status_, + task.ttl_del_cnt_, task.max_version_del_cnt_, + task.scan_cnt_, task.ret_code_.length(), task.ret_code_.ptr()))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else if (OB_FAIL(sql_append_hex_escape_str(task.row_key_, sql))) { + LOG_WARN("fail to append rowkey", K(ret)); + } else if (OB_FAIL(sql.append(")"))) { + LOG_WARN("fail to append"); + } else if (OB_FAIL(proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affect_rows))) { + LOG_WARN("fail to execute sql", K(ret), K(sql)); + } else { + LOG_INFO("success to execute sql", K(ret), K(sql)); + } + + return ret; +} + +// example: kv_attributes = {hbase: {maxversions: 3}} +int ObTTLUtil::parse_kv_attributes(const ObString &kv_attributes, int32_t &max_versions, int32_t &time_to_live) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator; + json::Parser json_parser; + json::Value *ast = nullptr; + if (kv_attributes.empty()) { + // skip + } else if (OB_FAIL(json_parser.init(&allocator))) { + LOG_WARN("failed to init json parser", K(ret)); + } else if (OB_FAIL(json_parser.parse(kv_attributes.ptr(), kv_attributes.length(), ast))) { + LOG_WARN("failed to parse kv attributes", K(ret), K(kv_attributes)); + } else if (NULL != ast + && ast->get_type() == json::JT_OBJECT + && ast->get_object().get_size() == 1) { + json::Pair *kv = ast->get_object().get_first(); + if (NULL != kv && kv != ast->get_object().get_header()) { + if (kv->name_.case_compare("HBASE") == 0) { + ast = kv->value_; + if (NULL == ast) { + // do nothing + } else if (ast->get_type() == json::JT_OBJECT) { + DLIST_FOREACH(elem, ast->get_object()) { + if (elem->name_.case_compare("TimeToLive") == 0) { + json::Value *ttl_val = elem->value_; + if (NULL != ttl_val && ttl_val->get_type() == json::JT_NUMBER) { + if (ttl_val->get_number() <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("time to live should greater than 0", K(ret), K(ttl_val)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "time to live, which should greater than 0"); + } else { + time_to_live = static_cast(ttl_val->get_number()); + } + } + } else if (elem->name_.case_compare("MaxVersions") == 0) { + json::Value *max_versions_val = elem->value_; + if (NULL != max_versions_val && max_versions_val->get_type() == json::JT_NUMBER) { + if (max_versions_val->get_number() <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("max versions should greater than 0", K(ret), K(max_versions_val)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "max versions, which should greater than 0"); + } else { + max_versions = static_cast(max_versions_val->get_number()); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported kv attribute", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes with wrong format"); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported kv attribute", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes with wrong format"); + } + } // end foreach + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported kv attribute", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes with wrong format"); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("only hbase mode is supported currently", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes with wrong format"); + } + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("not supported kv attribute", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes with wrong format"); + } + return ret; +} + +int ObTTLUtil::dispatch_ttl_cmd(const ObTTLParam ¶m) +{ + int ret = OB_SUCCESS; + int final_ret = OB_SUCCESS; + ObSEArray ttl_info_array; + if (OB_UNLIKELY(!param.is_valid() + || (!param.ttl_all_ && param.ttl_info_array_.empty()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(param), KR(ret)); + } else if (OB_FAIL(get_ttl_info(param, ttl_info_array))) { + LOG_WARN("fail to get tenant id", KR(ret), K(param)); + } else if (!ttl_info_array.empty()) { + const int64_t ttl_info_count = ttl_info_array.count(); + for (int i = 0; i < ttl_info_count && OB_SUCC(ret); ++i) { + const uint64_t tenant_id = ttl_info_array.at(i).tenant_id_; + if (OB_FAIL(dispatch_one_tenant_ttl(param.type_, *param.transport_, ttl_info_array.at(i)))) { + LOG_WARN("fail dispatch one tenant ttl", KR(ret), K(ttl_info_count), "ttl_info", ttl_info_array.at(i)); + } + } + } + return ret; +} + +int ObTableTTLChecker::init(const schema::ObTableSchema &table_schema, bool in_full_column_order) +{ + int ret = OB_SUCCESS; + int64_t tenant_id = table_schema.get_tenant_id(); + bool has_datetime_col = false; + if (tenant_id == OB_INVALID_TENANT_ID) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", K(ret), K(tenant_id)); + } else { + tenant_id_ = tenant_id; + ObString ttl_definition = table_schema.get_ttl_definition(); + if (ttl_definition.empty()) { + // do nothing + } else { + ObString right = ttl_definition; + bool is_end = false; + int64_t i = 0; + while (OB_SUCC(ret) && !is_end) { + ObString left = right.split_on(','); + if (left.empty()) { + left = right; + is_end = true; + } + ObTableTTLExpr ttl_expr; + ObString column_str = left.split_on('+').trim(); + left = left.trim(); + left += strlen("INTERVAL"); + left = left.trim(); + ObString interval_str = left.split_on(' '); + left.trim(); + ObString time_unit_str = left; + + ttl_expr.column_name_ = column_str; + ttl_expr.interval_ = atol(interval_str.ptr()); + if (time_unit_str.case_compare("SECOND") == 0) { + ttl_expr.time_unit_ = ObTableTTLTimeUnit::SECOND; + } else if (time_unit_str.case_compare("MINUTE") == 0) { + ttl_expr.time_unit_ = ObTableTTLTimeUnit::MINUTE; + } else if (time_unit_str.case_compare("HOUR") == 0) { + ttl_expr.time_unit_ = ObTableTTLTimeUnit::HOUR; + } else if (time_unit_str.case_compare("DAY") == 0) { + ttl_expr.time_unit_ = ObTableTTLTimeUnit::DAY; + } else if (time_unit_str.case_compare("MONTH") == 0) { + ttl_expr.time_unit_ = ObTableTTLTimeUnit::MONTH; + } else if (time_unit_str.case_compare("YEAR") == 0) { + ttl_expr.time_unit_ = ObTableTTLTimeUnit::YEAR; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unepxected time unit", K(ret)); + } + + // 2. get delta second and month + int64_t nsecond = 0; + int64_t nmonth = 0; + switch (ttl_expr.time_unit_) { + case ObTableTTLTimeUnit::SECOND: { + nsecond = ttl_expr.interval_; + break; + } + case ObTableTTLTimeUnit::MINUTE: { + nsecond = ttl_expr.interval_ * 60; + break; + } + case ObTableTTLTimeUnit::HOUR: { + nsecond = ttl_expr.interval_ * 60 * 60; + break; + } + case ObTableTTLTimeUnit::DAY: { + nsecond = ttl_expr.interval_ * 60 * 60 * 24; + break; + } + case ObTableTTLTimeUnit::MONTH: { + nmonth = ttl_expr.interval_; + break; + } + case ObTableTTLTimeUnit::YEAR: { + nmonth = ttl_expr.interval_ * 12; + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected time unit", K(ret), K_(ttl_expr.time_unit)); + } + + + if (OB_SUCC(ret)) { + ttl_expr.nsecond_ = nsecond; + ttl_expr.nmonth_ = nmonth; + if (ttl_expr.column_name_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null column name", K(ret)); + } else if (OB_FAIL(ttl_definition_.push_back(ttl_expr))) { + LOG_WARN("fail to add ttl expr", K(ret), K(ttl_expr)); + } else if (in_full_column_order) { + schema::ObTableSchema::const_column_iterator iter = table_schema.column_begin(); + schema::ObTableSchema::const_column_iterator end = table_schema.column_end(); + const schema::ObColumnSchemaV2 *col_schema = nullptr; + bool find_col = false; + for (int idx = 0; OB_SUCC(ret) && iter != end && !find_col; ++iter, idx++) { + col_schema = *iter; + if (OB_ISNULL(col_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column schema", K(ret)); + } else if (ttl_expr.column_name_.case_compare(col_schema->get_column_name_str()) == 0) { + find_col = true; + if (OB_FAIL(row_cell_ids_.push_back(idx))) { + LOG_WARN("fail to push back", K(ret), K(idx)); + } else if (col_schema->get_data_type() == ObDateTimeType) { + has_datetime_col = true; + } + } + } + } + } + } + } + } + + if (OB_SUCC(ret) && has_datetime_col) { + ObSchemaGetterGuard schema_guard; + ObTimeZoneInfoWrap tz_info_wrap; + const ObSysVariableSchema *sys_variable_schema = nullptr; + const ObSysVarSchema *system_timezone = nullptr; + ObTZMapWrap tz_map_wrap; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("get schema guard failed", K(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_sys_variable_schema(tenant_id, sys_variable_schema))) { + LOG_WARN("get sys variable schema failed", K(ret), K(tenant_id)); + } else if (NULL == sys_variable_schema) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys variable schema is NULL", K(ret)); + } else if (OB_FAIL(sys_variable_schema->get_sysvar_schema(SYS_VAR_TIME_ZONE, system_timezone))) { + LOG_WARN("fail to get system timezone", K(ret)); + } else if (OB_FAIL(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap))) { + LOG_WARN("get tenant timezone map failed", K(ret), K(tenant_id)); + } else if (OB_FAIL(tz_info_wrap_.init_time_zone(system_timezone->get_value(), OB_INVALID_VERSION, const_cast(*tz_map_wrap.get_tz_map())))) { + LOG_WARN("fail to init time zone info wrap", K(ret), K(system_timezone->get_value())); + } + } + + return ret; +} + +int ObTableTTLChecker::check_row_expired(const common::ObNewRow &row, bool &is_expired) +{ + int ret = OB_SUCCESS; + is_expired = false; + for (int i = 0; OB_SUCC(ret) && !is_expired && i < ttl_definition_.count(); i++) { + ObTableTTLExpr ttl_expr = ttl_definition_.at(i); + ObObj column = row.get_cell(row_cell_ids_.at(i)); + int64_t column_ts = column.get_timestamp(); + if (column.is_null()) { + continue; + } else if (column.get_type() == ObDateTimeType) { + // todo: get tz info from system var + const ObTimeZoneInfo *tz_info = tz_info_wrap_.get_time_zone_info(); + if (OB_FAIL(ObTimeConverter::datetime_to_timestamp(column_ts, tz_info, column_ts))) { + LOG_WARN("fail to convert datetime to utc ts", K(ret)); + } + } + + if (OB_SUCC(ret)) { + int64_t expire_ts = column_ts; + int64_t cur_ts = ObTimeUtility::current_time(); + if (ttl_expr.nsecond_ > 0 && OB_FAIL(ObTimeConverter::date_add_nsecond(column_ts, ttl_expr.nsecond_, 0, expire_ts))) { + LOG_WARN("fail to add nsecond", K(ret), K(column_ts), K(ttl_expr.nsecond_)); + } else if (ttl_expr.nsecond_ > 0 && OB_FAIL(ObTimeConverter::date_add_nmonth(column_ts, ttl_expr.nmonth_, expire_ts, true))) { + LOG_WARN("fail to add month", K(ret), K(column_ts), K(ttl_expr.nmonth_)); + } else if (expire_ts <= cur_ts) { + is_expired = true; + } + } + } + return ret; +} + +int ObTTLParam::add_ttl_info(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObSimpleTTLInfo info(tenant_id); + if (OB_FAIL(ttl_info_array_.push_back(info))) { + LOG_WARN("fail to push_back", K(ret), K(info)); + } + return ret; +} + +int ObTTLUtil::get_ttl_info(const ObTTLParam ¶m, ObIArray &ttl_info_array) +{ + int ret = OB_SUCCESS; + + ObArray tmp_info_array; + if (param.ttl_all_) { + if (OB_FAIL(get_all_user_tenant_ttl(tmp_info_array))) { + LOG_WARN("fail to get all tenant ttl info", KR(ret)); + } + } else { + if (OB_FAIL(tmp_info_array.assign(param.ttl_info_array_))) { + LOG_WARN("fail to assign", K(param), KR(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (tmp_info_array.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ttl info array should not be empty", KR(ret), K(param)); + } else { + const int64_t info_cnt = tmp_info_array.count(); + for (int64_t i = 0; OB_SUCC(ret) && (i < info_cnt); ++i) { + share::ObAllTenantInfo tenant_info; + bool is_restore = false; + const uint64_t tenant_id = tmp_info_array.at(i).tenant_id_; + if (OB_FAIL(share::schema::ObMultiVersionSchemaService::get_instance(). + check_tenant_is_restore(NULL, tenant_id, is_restore))) { + LOG_WARN("fail to check tenant is restore", KR(ret), K(i), "ttl_info", tmp_info_array.at(i)); + } else if (is_restore) { + LOG_INFO("skip restoring tenant to do ttl task", K(tenant_id)); + } else if (OB_FAIL(share::ObAllTenantInfoProxy::load_tenant_info(tenant_id, GCTX.sql_proxy_, + false, tenant_info))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; // ignore ret, so as to process the next tenant + LOG_WARN("tenant may be deleted, skip ttl task for this tenant", K(tenant_id)); + } else { + LOG_WARN("fail to load tenant info", KR(ret), K(tenant_id)); + } + } else if (tenant_info.is_standby()) { // Skip major freeze for standby tenants + LOG_INFO("skip do ttl task for standby tenant", K(tenant_info)); + } else if (OB_FAIL(ttl_info_array.push_back(tmp_info_array.at(i)))) { + LOG_WARN("fail to push back ttl info", KR(ret), K(i), "ttl_info", tmp_info_array.at(i)); + } + } + } + + return ret; +} + +int ObTTLUtil::dispatch_one_tenant_ttl(obrpc::ObTTLRequestArg::TTLRequestType type, + const rpc::frame::ObReqTransport &transport, + const ObSimpleTTLInfo &ttl_info) +{ + int ret = OB_SUCCESS; + if (!ttl_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ttl_info)); + } else { + const int64_t launch_start_time = ObTimeUtility::current_time(); + obrpc::ObSrvRpcProxy proxy; + ObAddr leader; + obrpc::ObTTLRequestArg req; + obrpc::ObTTLResponseArg resp; + uint64_t tenant_id = ttl_info.tenant_id_; + req.tenant_id_ = tenant_id; + req.cmd_code_ = type; + req.trigger_type_ = TRIGGER_TYPE::USER_TRIGGER; + if (OB_ISNULL(GCTX.location_service_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid GCTX", KR(ret)); + } else if (OB_FAIL(proxy.init(&transport))) { + LOG_WARN("fail to init", KR(ret)); + } else { + const int64_t MAX_RETRY_COUNT = 5; + bool ttl_done = false; + static const int64_t MAX_PROCESS_TIME_US = 10 * 1000 * 1000L; + for (int64_t i = 0; OB_SUCC(ret) && (!ttl_done) && (i < MAX_RETRY_COUNT); ++i) { + if (OB_FAIL(GCTX.location_service_->get_leader_with_retry_until_timeout(GCONF.cluster_id, + tenant_id, share::SYS_LS, leader))) { + LOG_WARN("fail to get ls locaiton leader", KR(ret), K(tenant_id)); + } else if (OB_FAIL(proxy.to(leader) + .trace_time(true) + .max_process_handler_time(MAX_PROCESS_TIME_US) + .by(tenant_id) + .dst_cluster_id(GCONF.cluster_id) + .dispatch_ttl(req, resp))) { + LOG_WARN("tenant ttl rpc failed", KR(ret), K(tenant_id), K(leader), K(ttl_info)); + } else if (FALSE_IT(ret = resp.err_code_)) { + } else if (OB_FAIL(ret)) { + if (OB_LEADER_NOT_EXIST == ret || OB_EAGAIN == ret) { + const int64_t RESERVED_TIME_US = 600 * 1000; // 600 ms + const int64_t timeout_remain_us = THIS_WORKER.get_timeout_remain(); + const int64_t idle_time_us = 200 * 1000 * (i + 1); + if (timeout_remain_us - idle_time_us > RESERVED_TIME_US) { + LOG_WARN("leader may switch or ddl confilict, will retry", KR(ret), K(tenant_id), K(ttl_info), + "ori_leader", leader, K(timeout_remain_us), K(idle_time_us), K(RESERVED_TIME_US)); + USLEEP(static_cast(idle_time_us)); + ret = OB_SUCCESS; + } else { + LOG_WARN("leader may switch or ddl confilict, will not retry cuz timeout_remain is " + "not enough", KR(ret), K(tenant_id), K(ttl_info), "ori_leader", leader, + K(timeout_remain_us), K(idle_time_us), K(RESERVED_TIME_US)); + } + } + } else { + ttl_done = true; + } + } + + if (OB_SUCC(ret) && !ttl_done) { + ret = OB_EAGAIN; + LOG_WARN("fail to retry ttl cuz switching role", KR(ret), K(MAX_RETRY_COUNT)); + } + } + + const int64_t launch_cost_time = ObTimeUtility::current_time() - launch_start_time; + LOG_INFO("do tenant ttl", KR(ret), K(tenant_id), K(leader), K(ttl_info), K(launch_cost_time)); + } + return ret; +} + +int ObTTLUtil::get_all_user_tenant_ttl(ObIArray &ttl_info_array) +{ + int ret = OB_SUCCESS; + ObSEArray tenant_ids; + { + share::schema::ObSchemaGetterGuard schema_guard; + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid GCTX", KR(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) { + LOG_WARN("fail to get tenant ids", KR(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.count(); ++i) { + if (is_user_tenant(tenant_ids[i])) { + ObSimpleTTLInfo info(tenant_ids[i]); + if(OB_FAIL(ttl_info_array.push_back(info))) { + LOG_WARN("fail to push back", KR(ret), "tenant_id", tenant_ids[i]); + } + } + } + return ret; +} + +int ObTTLUtil::get_tenant_table_ids(const uint64_t tenant_id, ObIArray &table_id_array) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard schema_guard; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_table_ids_in_tenant(tenant_id, table_id_array))) { + LOG_WARN("fail to get table ids in tenant", KR(ret), K(tenant_id)); + } + return ret; +} + +int ObTTLUtil::check_is_ttl_table(const ObTableSchema &table_schema, bool &is_ttl_table) +{ + int ret = OB_SUCCESS; + is_ttl_table = false; + if (table_schema.is_user_table() && !table_schema.is_in_recyclebin() && + (!table_schema.get_kv_attributes().empty() || !table_schema.get_ttl_definition().empty())) { + is_ttl_table = true; + } + return ret; +} + +int ObTTLUtil::check_ttl_task_exists(uint64_t tenant_id, common::ObISQLClient& proxy, + const uint64_t& task_id, const uint64_t& table_id, + ObTabletID& tablet_id, bool &is_exists) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + uint64_t result_cnt = 0; + if (OB_FAIL(sql.assign_fmt("SELECT (SELECT COUNT(*) FROM %s WHERE table_id = %ld" + " AND tablet_id = %ld AND task_id = %ld) + (SELECT COUNT(*) FROM %s WHERE" + " table_id = %ld AND tablet_id = %ld AND task_id = %ld) AS cnt", + share::OB_ALL_KV_TTL_TASK_HISTORY_TNAME, table_id, tablet_id.id(), task_id, + share::OB_ALL_KV_TTL_TASK_TNAME, table_id, tablet_id.id(), task_id))) { + LOG_WARN("sql assign fmt failed", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult* result = nullptr; + if (OB_FAIL(proxy.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + LOG_WARN("fail to execute sql", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected, query result must not be NULL", K(ret)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("fail to get next row", K(ret)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "cnt", result_cnt, uint64_t); + } + } + } + + if (OB_SUCC(ret)) { + if (result_cnt > 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected ttl task record count", KR(ret), K(tenant_id), K(task_id), K(table_id), K(tablet_id)); + } else { + is_exists = (result_cnt > 0); + } + } + + return ret; +} + + +} // end namespace rootserver +} // end namespace oceanbase \ No newline at end of file diff --git a/src/share/table/ob_ttl_util.h b/src/share/table/ob_ttl_util.h new file mode 100644 index 0000000000..dd17ab6ad1 --- /dev/null +++ b/src/share/table/ob_ttl_util.h @@ -0,0 +1,401 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SHARE_TABLE_OB_TABLE_TTL_UTIL_ +#define OCEANBASE_SHARE_TABLE_OB_TABLE_TTL_UTIL_ + +#include "lib/mysqlclient/ob_mysql_proxy.h" +#include "share/ob_srv_rpc_proxy.h" +#include "rootserver/ob_rs_async_rpc_proxy.h" +#include "rootserver/ob_server_manager.h" +#include "rootserver/ob_unit_manager.h" + +namespace oceanbase +{ +namespace common +{ + +#define OB_TTL_RESPONSE_MASK (1 << 5) +#define OB_TTL_STATUS_MASK (OB_TTL_RESPONSE_MASK - 1) + +#define SET_TASK_PURE_STATUS(status, state) ((status) = ((state) & OB_TTL_STATUS_MASK) + ((status & OB_TTL_RESPONSE_MASK))) +#define SET_TASK_RESPONSE(status, state) ((status) |= (((state) & 1) << 5)) +#define SET_TASK_STATUS(status, pure_status, is_responsed) { SET_TASK_PURE_STATUS(status, pure_status), SET_TASK_RESPONSE(status, is_responsed); } + +#define EVAL_TASK_RESPONSE(status) (((status) & OB_TTL_RESPONSE_MASK) >> 5) +#define EVAL_TASK_PURE_STATUS(status) (static_cast((status) & OB_TTL_STATUS_MASK)) + + +enum TRIGGER_TYPE +{ + PERIODIC_TRIGGER = 0, + USER_TRIGGER = 1, +}; + +enum ObTTLTaskType +{ + OB_TTL_TRIGGER, // todo:weiyouchao.wyc merge with rpc arg define + OB_TTL_SUSPEND, + OB_TTL_RESUME, + OB_TTL_CANCEL, + OB_TTL_MOVE, + OB_TTL_INVALID +}; + +enum ObTTLTaskStatus +{ + // for obsever + OB_TTL_TASK_PREPARE = 0, //inner state + OB_TTL_TASK_RUNNING = 1, + OB_TTL_TASK_PENDING = 2, + OB_TTL_TASK_CANCEL = 3, + OB_TTL_TASK_FINISH = 4, //inner state + OB_TTL_TASK_MOVING = 5, // deprecated + // for rs + OB_RS_TTL_TASK_CREATE = 15, + OB_RS_TTL_TASK_SUSPEND = 16, + OB_RS_TTL_TASK_CANCEL = 17, + OB_RS_TTL_TASK_MOVE = 18, + + OB_TTL_TASK_INVALID +}; + +typedef struct ObTTLStatus { + int64_t gmt_create_; + int64_t gmt_modified_; + + uint64_t tenant_id_; + uint64_t table_id_; + uint64_t tablet_id_; + int64_t task_id_; + + int64_t task_start_time_; + int64_t task_update_time_; + int64_t trigger_type_; + int64_t status_; + + uint64_t ttl_del_cnt_; + uint64_t max_version_del_cnt_; + uint64_t scan_cnt_; + ObString row_key_; + ObString ret_code_; + ObTTLStatus() + : gmt_create_(0), + gmt_modified_(0), + tenant_id_(OB_INVALID_ID), + table_id_(OB_INVALID_ID), + tablet_id_(OB_INVALID_ID), + task_id_(0), + task_start_time_(0), + task_update_time_(0), + trigger_type_(static_cast(PERIODIC_TRIGGER)), + status_(OB_TTL_TASK_INVALID), + ttl_del_cnt_(0), + max_version_del_cnt_(0), + scan_cnt_(0), + row_key_(), + ret_code_("OB_SUCCESS") {} + + TO_STRING_KV(K_(gmt_create), + K_(gmt_modified), + K_(tenant_id), + K_(table_id), + K_(tablet_id), + K_(task_id), + K_(task_start_time), + K_(task_update_time), + K_(trigger_type), + K_(status), + K_(ttl_del_cnt), + K_(max_version_del_cnt), + K_(scan_cnt), + K_(row_key), + K_(ret_code)); +} ObTTLStatus; + +typedef common::ObArray ObTTLStatusArray; + +typedef struct ObTTLStatusKey +{ + uint64_t tenant_id_; + uint64_t table_id_; + uint64_t tablet_id_; + uint64_t task_id_; + uint64_t partition_cnt_; + ObTTLStatusKey() + : tenant_id_(OB_INVALID_ID), + table_id_(OB_INVALID_ID), + tablet_id_(OB_INVALID_ID), + task_id_(0), + partition_cnt_(OB_INVALID_ID) + {} + + ObTTLStatusKey(uint64_t tenant_id, + uint64_t table_id, + uint64_t tablet_id, + uint64_t task_id) + : tenant_id_(tenant_id), + table_id_(table_id), + tablet_id_(tablet_id), + task_id_(task_id), + partition_cnt_(OB_INVALID_ID) + {} + + TO_STRING_KV(K_(tenant_id), + K_(table_id), + K_(tablet_id), + K_(task_id)); +} ObTTLStatusKey; + +typedef struct ObTTLStatusField { + ObString field_name_; + enum { + INT_TYPE = 0, + UINT_TYPE, + STRING_TYPE, + } type_; + + union data { + int64_t int_; + uint64_t uint_; + ObString str_; + data () : str_() {} + } data_; + ObTTLStatusField() + : field_name_(), + type_(INT_TYPE), + data_() {} + TO_STRING_KV(K_(field_name), K_(type)); +} ObTTLStatusField; + + +typedef common::ObArray ObTTLStatusFieldArray; + +typedef struct ObTTLDayTime { + int32_t hour_; + int32_t min_; + int32_t sec_; + ObTTLDayTime() + : hour_(0), min_(0), sec_(0) {} + bool is_valid() { + return ((hour_ >= 0 && hour_ <= 24) && + (min_ >= 0 && min_ <= 60) && + (sec_ >= 0 && sec_ <= 60)); + } + TO_STRING_KV(K_(hour), K_(min), K_(sec)); +} ObTTLDayTime; + +struct ObTTLDutyDuration +{ + ObTTLDutyDuration() + : begin_(), end_(), not_set_(true) {} + + bool is_valid() { + return ((begin_.is_valid() && end_.is_valid()) || not_set_); + } + TO_STRING_KV(K_(begin), K_(end)); + + ObTTLDayTime begin_, end_; + bool not_set_; +}; + +class ObTTLTime { +public: + static int64_t current_time(); + + static bool is_same_day(int64_t ttl_time1, + int64_t ttl_time2); +}; + +struct ObSimpleTTLInfo +{ +public: + uint64_t tenant_id_; + + ObSimpleTTLInfo() + : tenant_id_(OB_INVALID_TENANT_ID) + {} + + ObSimpleTTLInfo(const uint64_t tenant_id) + : tenant_id_(tenant_id) + {} + + bool is_valid() const { return (OB_INVALID_TENANT_ID != tenant_id_); } + TO_STRING_KV(K_(tenant_id)); + OB_UNIS_VERSION(1); +}; + +struct ObTTLParam +{ +public: + ObTTLParam() + : ttl_info_array_(), ttl_all_(false), transport_(nullptr) + {} + + void reset() + { + ttl_info_array_.reset(); + ttl_all_ = false; + transport_ = nullptr; + } + + bool is_valid() const + { + return (nullptr != transport_); + } + + int add_ttl_info(const uint64_t tenant_id); + + TO_STRING_KV(K_(ttl_info_array), K_(ttl_all), KP_(transport)); + + common::ObArray ttl_info_array_; + bool ttl_all_; + rpc::frame::ObReqTransport *transport_; + obrpc::ObTTLRequestArg::TTLRequestType type_; +}; + +class ObTTLUtil { +public: + static int parse(const char* str, ObTTLDutyDuration& duration); + static bool current_in_duration(ObTTLDutyDuration& duration); + + static int insert_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatus& task); + + static int replace_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatus& task); + + static int update_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatusKey& key, + ObTTLStatusFieldArray& update_fields); + + static int update_ttl_task_all_fields(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatus& update_task); + + static int delete_ttl_task(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatusKey& key, + int64_t &affect_rows); + + static int read_ttl_tasks(uint64_t tenant_id, + const char* tname, + common::ObISQLClient& proxy, + ObTTLStatusFieldArray& filters, + ObTTLStatusArray& result_arr, + bool for_update = false, + common::ObIAllocator *allocator = NULL); + + static int read_tenant_ttl_task(uint64_t tenant_id, + common::ObISQLClient& proxy, + ObTTLStatus &ttl_record, + common::ObIAllocator *allocator = NULL); + + static int move_task_to_history_table(uint64_t tenant_id, uint64_t task_id, + common::ObMySQLTransaction& proxy, + int64_t batch_size, int64_t &move_rows); + + static int move_tenant_task_to_history_table(uint64_t tenant_id, uint64_t task_id, + common::ObMySQLTransaction& proxy); + + + static bool check_can_do_work(); + static bool check_can_process_tenant_tasks(uint64_t tenant_id); + + static int parse_kv_attributes(const ObString &kv_attributes, int32_t &max_versions, int32_t &time_to_live); + + static int dispatch_ttl_cmd(const ObTTLParam ¶m); + static int get_ttl_info(const ObTTLParam ¶m, ObIArray &ttl_info_array); + + static int check_is_ttl_table(const ObTableSchema &table_schema, bool &is_ttl_table); + static int get_tenant_table_ids(const uint64_t tenant_id, common::ObIArray &table_id_array); + static int check_ttl_task_exists(uint64_t tenant_id, common::ObISQLClient& proxy, + const uint64_t& task_id, const uint64_t& table_id, + ObTabletID& tablet_id, bool &is_exists); + + const static uint64_t TTL_TENNAT_TASK_TABLET_ID = -1; + const static uint64_t TTL_TENNAT_TASK_TABLE_ID = -1; +private: + static bool extract_val(const char* ptr, uint64_t len, int& val); + static bool valid_digit(const char* ptr, uint64_t len); + static int parse_ttl_daytime(ObString& in, ObTTLDayTime& daytime); + static int dispatch_one_tenant_ttl(obrpc::ObTTLRequestArg::TTLRequestType type, + const rpc::frame::ObReqTransport &transport, + const ObSimpleTTLInfo &ttl_info); + static int get_all_user_tenant_ttl(common::ObIArray &ttl_info_array); + DISALLOW_COPY_AND_ASSIGN(ObTTLUtil); +}; + +enum class ObTableTTLTimeUnit +{ + INVALID, + SECOND, + MINUTE, + HOUR, + DAY, + MONTH, + YEAR +}; + +class ObTableTTLExpr +{ +public: + ObTableTTLExpr(): column_name_(), interval_(), time_unit_(ObTableTTLTimeUnit::INVALID), nsecond_(0), nmonth_(0), is_negative_(false) {} + ~ObTableTTLExpr() {} + const ObString &get_ttl_column() const { return column_name_; } + TO_STRING_KV(K_(column_name), K_(interval), K_(time_unit)); +public: + ObString column_name_; + int64_t interval_; + ObTableTTLTimeUnit time_unit_; + int64_t nsecond_; + int64_t nmonth_; + bool is_negative_; +}; + +class ObTableTTLChecker +{ +public: + ObTableTTLChecker() + : ttl_definition_(), + row_cell_ids_(), + tenant_id_(common::OB_INVALID_TENANT_ID) + { + ttl_definition_.set_attr(ObMemAttr(MTL_ID(), "TTLCheckerDef")); + row_cell_ids_.set_attr(ObMemAttr(MTL_ID(), "TTLCheckerCells")); + } + ~ObTableTTLChecker() {} + // init ttl checker with table schema, if in_full_column_order is true, the checked row + // should be in full column schema order, or you shoud set ttl cell ids explicitly. + int init(const share::schema::ObTableSchema &table_schema, bool in_full_column_order = true); + int check_row_expired(const common::ObNewRow &row, bool &is_expired); + const common::ObIArray &get_ttl_definition() const { return ttl_definition_; } + common::ObIArray &get_row_cell_ids() { return row_cell_ids_; } +private: + common::ObSEArray ttl_definition_; + common::ObSEArray row_cell_ids_; // cell idx scaned row for each ttl expr + int64_t tenant_id_; + ObTimeZoneInfoWrap tz_info_wrap_; +}; + +} // end namespace rootserver +} // end namespace oceanbase + +#endif /* OCEANBASE_SHARE_TABLE_OB_TABLE_TTL_UTIL_ */ \ No newline at end of file diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index 4ed3828390..b5aae79082 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -45,6 +45,7 @@ #include "rpc/obmysql/ob_sql_sock_session.h" #include "sql/plan_cache/ob_plan_cache.h" #include "sql/plan_cache/ob_ps_cache.h" +#include "share/table/ob_ttl_util.h" namespace oceanbase { using namespace common; @@ -158,7 +159,7 @@ int ObFreezeExecutor::execute(ObExecContext &ctx, ObFreezeStmt &stmt) param.freeze_all_user_ = stmt.is_freeze_all_user(); param.freeze_all_meta_ = stmt.is_freeze_all_meta(); param.transport_ = GCTX.net_frame_->get_req_transport(); - for (int64_t i = 0; (i < stmt.get_tenant_ids().count()) && OB_SUCC(ret); ++i) { + for (int64_t i = 0; i < stmt.get_tenant_ids().count() && OB_SUCC(ret); ++i) { uint64_t tenant_id = stmt.get_tenant_ids().at(i); if (OB_FAIL(param.add_freeze_info(tenant_id))) { LOG_WARN("fail to assign", KR(ret), K(tenant_id)); @@ -2606,6 +2607,44 @@ int ObCancelRestoreExecutor::execute(ObExecContext &ctx, ObCancelRestoreStmt &st return ret; } +int ObTableTTLExecutor::execute(ObExecContext& ctx, ObTableTTLStmt& stmt) +{ + int ret = OB_SUCCESS; + ObTaskExecutorCtx* task_exec_ctx = GET_TASK_EXECUTOR_CTX(ctx); + obrpc::ObCommonRpcProxy* common_rpc_proxy = NULL; + + if (OB_ISNULL(task_exec_ctx)) { + ret = OB_NOT_INIT; + LOG_WARN("get task executor context failed"); + } else if (OB_FAIL(task_exec_ctx->get_common_rpc(common_rpc_proxy))) { + LOG_WARN("get common rpc proxy failed", K(ret)); + } else if (OB_ISNULL(common_rpc_proxy)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("common_rpc_proxy is null", K(ret)); + } else { + FLOG_INFO("ObTableTTLExecutor::execute", K(stmt), K(ctx)); + common::ObTTLParam param; + ObSEArray ttl_info_array; + param.ttl_all_ = stmt.is_ttl_all(); + param.transport_ = GCTX.net_frame_->get_req_transport(); + param.type_ = stmt.get_type(); + for (int64_t i = 0; (i < stmt.get_tenant_ids().count()) && OB_SUCC(ret); i++) { + uint64_t tenant_id = stmt.get_tenant_ids().at(i); + if (OB_FAIL(param.add_ttl_info(tenant_id))) { + LOG_WARN("fail to assign ttl info", KR(ret), K(tenant_id)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_UNLIKELY(!param.ttl_all_ && param.ttl_info_array_.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(param), KR(ret)); + } else if (OB_FAIL(ObTTLUtil::dispatch_ttl_cmd(param))) { + LOG_WARN("fail to dispatch ttl cmd", K(ret), K(param)); + } + } + return ret; +} } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/cmd/ob_alter_system_executor.h b/src/sql/engine/cmd/ob_alter_system_executor.h index 76b6462e00..691673d2c4 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.h +++ b/src/sql/engine/cmd/ob_alter_system_executor.h @@ -129,6 +129,7 @@ DEF_SIMPLE_EXECUTOR(ObBackupSetEncryption); DEF_SIMPLE_EXECUTOR(ObBackupSetDecryption); DEF_SIMPLE_EXECUTOR(ObAddRestoreSource); DEF_SIMPLE_EXECUTOR(ObClearRestoreSource); +DEF_SIMPLE_EXECUTOR(ObTableTTL); DEF_SIMPLE_EXECUTOR(ObSetRegionBandwidth); diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 811db7df75..f5ce63e8dd 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -997,6 +997,10 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObCheckpointSlogStmt, ObCheckpointSlogExecutor); break; } + case stmt::T_TABLE_TTL: { + DEFINE_EXECUTE_CMD(ObTableTTLStmt, ObTableTTLExecutor); + break; + } case stmt::T_CS_DISKMAINTAIN: case stmt::T_TABLET_CMD: case stmt::T_SWITCH_ROOTSERVER: diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 46e30d1ce7..f3fedf4645 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -995,6 +995,8 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"wash", WASH}, {"query_response_time",QUERY_RESPONSE_TIME}, {"statement_id", STATEMENT_ID}, + {"TTL", TTL}, + {"kv_attributes", KV_ATTRIBUTES}, }; /** https://dev.mysql.com/doc/refman/5.7/en/sql-syntax-prepared-statements.html diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 027a8fabc5..827dce20da 100755 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -289,7 +289,7 @@ END_P SET_VAR DELIMITER JOB JSON JSON_ARRAYAGG JSON_OBJECTAGG JSON_VALUE JSON_TABLE - KEY_BLOCK_SIZE KEY_VERSION KVCACHE + KEY_BLOCK_SIZE KEY_VERSION KVCACHE KV_ATTRIBUTES LAG LANGUAGE LAST LAST_VALUE LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_ LISTAGG LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS @@ -344,7 +344,7 @@ END_P SET_VAR DELIMITER TABLE_CHECKSUM TABLE_MODE TABLE_ID TABLE_NAME TABLEGROUPS TABLES TABLESPACE TABLET TABLET_ID TABLET_MAX_SIZE TEMPLATE TEMPORARY TEMPTABLE TENANT TEXT THAN TIME TIMESTAMP TIMESTAMPADD TIMESTAMPDIFF TP_NO TP_NAME TRACE TRADITIONAL TRANSACTION TRIGGERS TRIM TRUNCATE TYPE TYPES TASK TABLET_SIZE - TABLEGROUP_ID TENANT_ID THROTTLE TIME_ZONE_INFO TOP_K_FRE_HIST TIMES TRIM_SPACE + TABLEGROUP_ID TENANT_ID THROTTLE TIME_ZONE_INFO TOP_K_FRE_HIST TIMES TRIM_SPACE TTL UNCOMMITTED UNDEFINED UNDO_BUFFER_SIZE UNDOFILE UNICODE UNINSTALL UNIT UNIT_GROUP UNIT_NUM UNLOCKED UNTIL UNUSUAL UPGRADE USE_BLOOM_FILTER UNKNOWN USE_FRM USER USER_RESOURCES UNBOUNDED UP UNLIMITED @@ -516,6 +516,7 @@ END_P SET_VAR DELIMITER %type opt_value_on_empty_or_error_or_mismatch opt_on_mismatch %type table_values_caluse table_values_caluse_with_order_by_and_limit values_row_list row_value +%type ttl_definition ttl_expr ttl_unit %start sql_stmt %% //////////////////////////////////////////////////////////////// @@ -6341,6 +6342,16 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void)($2) ; /* make bison mute */ malloc_non_terminal_node($$, result->malloc_pool_, T_EXTERNAL_FILE_PATTERN, 1, $3); } +| TTL '(' ttl_definition ')' +{ + merge_nodes($$, result, T_TTL_DEFINITION, $3); + dup_expr_string($$, result, @3.first_column, @3.last_column); +} +| KV_ATTRIBUTES opt_equal_mark STRING_VALUE +{ + (void)($2); /* make bison mute*/ + malloc_non_terminal_node($$, result->malloc_pool_, T_KV_ATTRIBUTES, 1, $3); +} ; parallel_option: @@ -6362,6 +6373,64 @@ PARALLEL opt_equal_mark INTNUM } ; +ttl_definition: +ttl_expr +{ + $$ = $1; +} +| ttl_definition ',' ttl_expr +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); +} +; + +ttl_expr: +simple_expr '+' INTERVAL INTNUM ttl_unit +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_TTL_EXPR, 3, $1, $4, $5); + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +; + +ttl_unit: +SECOND +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = DATE_UNIT_SECOND; + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +| MINUTE +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = DATE_UNIT_MINUTE; + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +| HOUR +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = DATE_UNIT_HOUR; + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +| DAY +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = DATE_UNIT_DAY; + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +| MONTH +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = DATE_UNIT_MONTH; + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +| YEAR +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = DATE_UNIT_YEAR; + dup_expr_string($$, result, @1.first_column, @1.last_column); +} +; + relation_name_or_string: relation_name { $$ = $1; $$->type_ = T_VARCHAR;} @@ -14234,6 +14303,10 @@ DROP CONSTRAINT constraint_name // merge_nodes(col_list, result->malloc_pool_, T_COLUMN_LIST, $3); // malloc_non_terminal_node($$, result->malloc_pool_, T_ORDER_BY, 1, col_list); // }*/ +| REMOVE TTL +{ + malloc_terminal_node($$, result->malloc_pool_, T_REMOVE_TTL); +} ; // mysql 模式下的 constraint 特指 check constraint @@ -15468,6 +15541,38 @@ ALTER SYSTEM RESUME BACKUP malloc_non_terminal_node($$, result->malloc_pool_, T_BACKUP_MANAGE, 2, type, value); } | +ALTER SYSTEM TRIGGER TTL opt_tenant_list_v2 +{ + ParseNode *type = NULL; + malloc_terminal_node(type, result->malloc_pool_, T_INT); + type->value_ = 0; + malloc_non_terminal_node($$, result->malloc_pool_, T_TABLE_TTL, 2, type, $5); +} +| +ALTER SYSTEM SUSPEND TTL opt_tenant_list_v2 +{ + ParseNode *type = NULL; + malloc_terminal_node(type, result->malloc_pool_, T_INT); + type->value_ = 1; + malloc_non_terminal_node($$, result->malloc_pool_, T_TABLE_TTL, 2, type, $5); +} +| +ALTER SYSTEM RESUME TTL opt_tenant_list_v2 +{ + ParseNode *type = NULL; + malloc_terminal_node(type, result->malloc_pool_, T_INT); + type->value_ = 2; + malloc_non_terminal_node($$, result->malloc_pool_, T_TABLE_TTL, 2, type, $5); +} +| +ALTER SYSTEM CANCEL TTL opt_tenant_list_v2 +{ + ParseNode *type = NULL; + malloc_terminal_node(type, result->malloc_pool_, T_INT); + type->value_ = 3; + malloc_non_terminal_node($$, result->malloc_pool_, T_TABLE_TTL, 2, type, $5); +} +| ALTER SYSTEM VALIDATE DATABASE opt_copy_id { ParseNode *type = NULL; @@ -18512,6 +18617,7 @@ ACCOUNT | TRIM | TRIM_SPACE | TRUNCATE +| TTL | TYPE | TYPES | TABLEGROUP_ID @@ -18598,6 +18704,7 @@ ACCOUNT | MY_NAME | CONNECT | STATEMENT_ID +| KV_ATTRIBUTES ; unreserved_keyword_special: diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index a12e7258fb..0b32d5ff96 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -1760,7 +1760,8 @@ int get_sys_tenant_alter_system_priv( stmt::T_BACKUP_CLEAN != basic_stmt->get_stmt_type() && stmt::T_DELETE_POLICY != basic_stmt->get_stmt_type() && stmt::T_BACKUP_KEY != basic_stmt->get_stmt_type() && - stmt::T_RECOVER != basic_stmt->get_stmt_type()) { + stmt::T_RECOVER != basic_stmt->get_stmt_type() && + stmt::T_TABLE_TTL != basic_stmt->get_stmt_type()) { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("Only sys tenant can do this operation", K(ret), "stmt type", basic_stmt->get_stmt_type()); diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 8b9a452073..42d7fc8536 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -4309,6 +4309,71 @@ int ObAlterSystemResolverUtil::get_tenant_ids(const ParseNode &t_node, ObIArray< return ret; } +int ObTableTTLResolver::resolve(const ParseNode& parse_tree) +{ + int ret = OB_SUCCESS; + uint64_t tenant_data_version = 0;; + const uint64_t cur_tenant_id = session_info_->get_effective_tenant_id(); + if (OB_FAIL(GET_MIN_DATA_VERSION(cur_tenant_id, tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("TTL command is not supported in data version less than 4.2.1", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "TTL command is not supported in data version less than 4.2.1"); + } else if (OB_UNLIKELY(T_TABLE_TTL != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_TABLE_TTL", "type", get_type_name(parse_tree.type_)); + } else if (OB_UNLIKELY(NULL == parse_tree.children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children should not be null", K(ret)); + } else if (OB_UNLIKELY(2 != parse_tree.num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children num not match", K(ret), "num_child", parse_tree.num_child_); + } else if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info should not be null", K(ret)); + } else { + ObTableTTLStmt* ttl_stmt = create_stmt(); + const int64_t type = parse_tree.children_[0]->value_; + ParseNode *opt_tenant_list_v2 = parse_tree.children_[1]; + if (NULL == ttl_stmt) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("create ObTableTTLStmt failed"); + } else if (OB_FAIL(ttl_stmt->set_type(type))) { + LOG_WARN("fail to set param", K(ret), K(type)); + } else { + if (NULL == opt_tenant_list_v2) { + if (OB_SYS_TENANT_ID == cur_tenant_id) { + ttl_stmt->set_ttl_all(true); + } else if (OB_FAIL(ttl_stmt->get_tenant_ids().push_back(cur_tenant_id))) { + LOG_WARN("fail to push owned tenant id ", KR(ret), "owned tenant_id", cur_tenant_id); + } + } else if (OB_SYS_TENANT_ID != cur_tenant_id) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("only sys tenant can add suffix opt(tenant=name)", KR(ret), K(cur_tenant_id)); + } else { + bool affect_all = false; + bool affect_all_user = false; + bool affect_all_meta = false; + if (OB_FAIL(Util::resolve_tenant(*opt_tenant_list_v2, cur_tenant_id, ttl_stmt->get_tenant_ids(), affect_all, affect_all_user, affect_all_meta))) { + LOG_WARN("fail to resolve tenant", KR(ret), KP(opt_tenant_list_v2), K(cur_tenant_id)); + } else if (affect_all_meta) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("affect_all_meta and affect_all_user is not supported", KR(ret), K(affect_all_meta), K(affect_all_user)); + LOG_USER_WARN(OB_NOT_SUPPORTED, "affect_all_meta and affect_all_user"); + } else if (affect_all || affect_all_user) { + ttl_stmt->set_ttl_all(true); + } + } + } + if (OB_SUCC(ret)) { + stmt_ = ttl_stmt; + } + } + + return ret; +} + int ObBackupManageResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index c81332b426..882e67075e 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -250,6 +250,7 @@ DEF_SIMPLE_CMD_RESOLVER(ObEnableSqlThrottleResolver); DEF_SIMPLE_CMD_RESOLVER(ObDisableSqlThrottleResolver); DEF_SIMPLE_CMD_RESOLVER(ObSetRegionBandwidthResolver); DEF_SIMPLE_CMD_RESOLVER(ObCancelRestoreResolver); +DEF_SIMPLE_CMD_RESOLVER(ObTableTTLResolver); #undef DEF_SIMPLE_CMD_RESOLVER diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index 7a3586cb6b..0c1dc0feb6 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -1173,6 +1173,47 @@ private: share::ObBackupPathString backup_dest_; ObString encrypt_key_; }; +class ObTableTTLStmt : public ObSystemCmdStmt { +public: + ObTableTTLStmt() + : ObSystemCmdStmt(stmt::T_TABLE_TTL), + type_(obrpc::ObTTLRequestArg::TTL_INVALID_TYPE), + opt_tenant_ids_(), + ttl_all_(false) + {} + virtual ~ObTableTTLStmt() + {} + + obrpc::ObTTLRequestArg::TTLRequestType get_type() const + { + return type_; + } + int set_type(const int64_t type) + { + int ret = common::OB_SUCCESS; + + if (type < 0 || type >= obrpc::ObTTLRequestArg::TTL_MOVE_TYPE) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid args", K(type)); + } else { + type_ = static_cast(type); + } + + return ret; + } + inline common::ObSArray &get_tenant_ids() { return opt_tenant_ids_; } + bool is_ttl_all() const { return ttl_all_; } + void set_ttl_all(bool ttl_all) { ttl_all_ = ttl_all; } + + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(tenant_id), K_(type), + K_(opt_tenant_ids), K_(ttl_all)); + +private: + uint64_t tenant_id_; + obrpc::ObTTLRequestArg::TTLRequestType type_; + common::ObSArray opt_tenant_ids_; + bool ttl_all_; +}; class ObBackupSetEncryptionStmt : public ObSystemCmdStmt { diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 09b12a94f1..75922f7841 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -412,6 +412,10 @@ int ObAlterTableResolver::set_table_options() SQL_RESV_LOG(WARN, "Write database_name to alter_table_schema failed!", K(database_name_), K(ret)); } else if (OB_FAIL(alter_table_schema.set_encryption_str(encryption_))) { SQL_RESV_LOG(WARN, "Write encryption to alter_table_schema failed!", K(encryption_), K(ret)); + } else if (OB_FAIL(alter_table_schema.set_ttl_definition(ttl_definition_))) { + SQL_RESV_LOG(WARN, "Write ttl_definition to alter_table_schema failed!", K(ret)); + } else if (OB_FAIL(alter_table_schema.set_kv_attributes(kv_attributes_))) { + SQL_RESV_LOG(WARN, "Write kv_attributes to alter_table_schema failed!", K(ret)); } else { alter_table_schema.alter_option_bitset_ = alter_table_bitset_; } @@ -611,6 +615,7 @@ int ObAlterTableResolver::resolve_action_list(const ParseNode &node) int64_t alter_column_times = 0; int64_t alter_column_visibility_times = 0; ObReducedVisibleColSet reduced_visible_col_set; + bool has_alter_column_option = false; //in mysql mode, resolve add index after resolve column actions ObSEArray add_index_action_idxs; for (int64_t i = 0; OB_SUCC(ret) && i < node.num_child_; ++i) { @@ -657,6 +662,7 @@ int ObAlterTableResolver::resolve_action_list(const ParseNode &node) alter_table_stmt->set_alter_table_column(); bool temp_is_modify_column_visibility = false; bool is_drop_column = false; + has_alter_column_option = true; if (OB_FAIL(resolve_column_options(*action_node, temp_is_modify_column_visibility, is_drop_column, reduced_visible_col_set))) { SQL_RESV_LOG(WARN, "Resolve column option failed!", K(ret)); } else { @@ -856,19 +862,26 @@ int ObAlterTableResolver::resolve_action_list(const ParseNode &node) } } break; - } + } case T_MODIFY_ALL_TRIGGERS: { alter_table_stmt->set_is_alter_triggers(true); if (OB_FAIL(resolve_modify_all_trigger(*action_node))) { SQL_RESV_LOG(WARN, "failed to resolve trigger option!", K(ret)); } - } - break; + break; + } case T_SET_INTERVAL: { - if (OB_FAIL(resolve_set_interval(alter_table_stmt, *action_node))) { - SQL_RESV_LOG(WARN, "failed to resolve foreign key options in mysql mode!", K(ret)); - } - break; + if (OB_FAIL(resolve_set_interval(alter_table_stmt, *action_node))) { + SQL_RESV_LOG(WARN, "failed to resolve foreign key options in mysql mode!", K(ret)); + } + break; + } + case T_REMOVE_TTL: { + ttl_definition_.reset(); + if (OB_FAIL(alter_table_bitset_.add_member(ObAlterTableArg::TTL_DEFINITION))) { + SQL_RESV_LOG(WARN, "failed to add member to bitset!", K(ret)); + } + break; } default: { ret = OB_ERR_UNEXPECTED; @@ -1022,6 +1035,38 @@ int ObAlterTableResolver::resolve_action_list(const ParseNode &node) if (OB_SUCC(ret) && OB_FAIL(check_alter_column_schemas_valid(*alter_table_stmt))) { LOG_WARN("failed to check alter column schemas valid", K(ret)); } + + if (OB_SUCC(ret)) { + // modify/change definition ttl column is not allowed currently + if (has_alter_column_option && alter_table_bitset_.has_member(ObAlterTableArg::TTL_DEFINITION)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "SET/REMOVE TTL together with other Alter Column DDL"); + } else if (has_alter_column_option) { + ObTableSchema tbl_schema; + ObSEArray ttl_columns; + if (OB_FAIL(get_table_schema_for_check(tbl_schema))) { + LOG_WARN("fail to get table schema", K(ret)); + } else if (OB_FAIL(get_ttl_columns(tbl_schema.get_ttl_definition(), ttl_columns))) { + LOG_WARN("fail to get ttl column", K(ret)); + } else if (ttl_columns.empty()) { + // do nothing + } else { + AlterTableSchema &alter_table_schema = get_alter_table_stmt()->get_alter_table_arg().alter_table_schema_; + ObTableSchema::const_column_iterator iter = alter_table_schema.column_begin(); + ObTableSchema::const_column_iterator end = alter_table_schema.column_end(); + for (; OB_SUCC(ret) && iter != end; ++iter) { + const AlterColumnSchema *column = static_cast(*iter); + if (OB_ISNULL(column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null alter column", K(ret)); + } else if (is_ttl_column(column->get_origin_column_name(), ttl_columns)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("Modify/Change TTL column is not allowed", K(ret)); + } + } + } + } + } } return ret; } @@ -5818,6 +5863,17 @@ int ObAlterTableResolver::resolve_modify_all_trigger(const ParseNode &node) return ret; } +bool ObAlterTableResolver::is_ttl_column(const ObString &orig_column_name, const ObIArray &ttl_columns) +{ + bool bret = false; + for (int64_t i = 0; i < ttl_columns.count() && !bret; i++) { + if (orig_column_name.case_compare(ttl_columns.at(i)) == 0) { + bret = true; + } + } + return bret; +} + int ObAlterTableResolver::check_mysql_rename_column(const AlterColumnSchema &alter_column_schema, const ObTableSchema &origin_table_schema, ObAlterTableStmt &alter_table_stmt) diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.h b/src/sql/resolver/ddl/ob_alter_table_resolver.h index ef954cff43..e2a4ada713 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.h +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.h @@ -175,6 +175,7 @@ private: const share::schema::ObColumnSchemaV2 &src_col_schema, const share::schema::ObColumnSchemaV2 &dst_col_schema); int generate_index_arg_cascade(); + bool is_ttl_column(const common::ObString &orig_column_name, const ObIArray &ttl_columns); int check_alter_column_schemas_valid(ObAlterTableStmt &stmt); diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index 0ade95be0a..c7ad995d14 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -2404,7 +2404,9 @@ int ObCreateTableResolver::set_table_option_to_schema(ObTableSchema &table_schem if (OB_FAIL(table_schema.set_expire_info(expire_info_)) || OB_FAIL(table_schema.set_compress_func_name(compress_method_)) || OB_FAIL(table_schema.set_comment(comment_)) || - OB_FAIL(table_schema.set_tablegroup_name(tablegroup_name_))) { + OB_FAIL(table_schema.set_tablegroup_name(tablegroup_name_)) || + OB_FAIL(table_schema.set_ttl_definition(ttl_definition_)) || + OB_FAIL(table_schema.set_kv_attributes(kv_attributes_))) { SQL_RESV_LOG(WARN, "set table_options failed", K(ret)); } } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index e4d5baa02f..3577408c0e 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -42,6 +42,7 @@ #include "sql/resolver/dcl/ob_dcl_resolver.h" #include "sql/engine/expr/ob_expr_lob_utils.h" #include "pl/ob_pl_stmt.h" +#include "share/table/ob_ttl_util.h" namespace oceanbase { using namespace common; @@ -109,7 +110,9 @@ ObDDLResolver::ObDDLResolver(ObResolverParams ¶ms) tablespace_id_(OB_INVALID_ID), table_dop_(DEFAULT_TABLE_DOP), hash_subpart_num_(-1), - is_external_table_(false) + is_external_table_(false), + ttl_definition_(), + kv_attributes_() { table_mode_.reset(); } @@ -2129,6 +2132,84 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool } break; } + case T_TTL_DEFINITION: { + uint64_t tenant_data_version = 0;; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl definition is not supported in data version less than 4.2.1", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ttl definition is not supported in data version less than 4.2.1"); + } else if (!is_index_option) { + if (OB_ISNULL(option_node->children_)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "(the children of option_node is null", K(option_node->children_), K(ret)); + } else { + ObRawExpr *expr = NULL; + ObString tmp_str; + tmp_str.assign_ptr(const_cast(option_node->str_value_), + static_cast(option_node->str_len_)); + if (OB_ISNULL(option_node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(ERROR,"children can't be null", K(ret)); + } else if (OB_FAIL(ob_write_string(*allocator_, tmp_str, ttl_definition_))) { + SQL_RESV_LOG(WARN, "write string failed", K(ret)); + } else if (OB_FAIL(check_ttl_definition(option_node))) { + LOG_WARN("fail to check ttl definition", K(ret)); + } else if (stmt::T_ALTER_TABLE == stmt_->get_stmt_type()) { + if (OB_FAIL(alter_table_bitset_.add_member(ObAlterTableArg::TTL_DEFINITION))) { + SQL_RESV_LOG(WARN, "failed to add member to bitset!", K(ret)); + } + } + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("index option should not specify ttl", K(ret)); + } + break; + } + case T_KV_ATTRIBUTES: { + uint64_t tenant_data_version = 0;; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("kv attributes is not supported in data version less than 4.2.1", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "kv attributes is not supported in data version less than 4.2.1"); + } else if (!is_index_option) { + if (OB_ISNULL(option_node->children_)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "(the children of option_node is null", K(option_node->children_), K(ret)); + } else if (OB_ISNULL(option_node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(ERROR,"children can't be null", K(ret)); + } else { + ObRawExpr *expr = NULL; + ObString tmp_str; + int32_t max_versions = 0; + int32_t time_to_live = 0; + tmp_str.assign_ptr(const_cast(option_node->children_[0]->str_value_), + static_cast(option_node->children_[0]->str_len_)); + LOG_INFO("resolve kv attributes", K(tmp_str)); + if (OB_FAIL(ObSQLUtils::convert_sql_text_to_schema_for_storing( + *allocator_, session_info_->get_dtc_params(), tmp_str))) { + LOG_WARN("fail to convert comment to utf8", K(ret)); + } else if (OB_FAIL(ObTTLUtil::parse_kv_attributes(tmp_str, max_versions, time_to_live))) { + LOG_WARN("fail to parse kv attributes", K(ret)); + } else if (OB_FAIL(ob_write_string(*allocator_, tmp_str, kv_attributes_))) { + SQL_RESV_LOG(WARN, "write string failed", K(ret)); + } else if (stmt::T_ALTER_TABLE == stmt_->get_stmt_type()) { + if (OB_FAIL(alter_table_bitset_.add_member(ObAlterTableArg::KV_ATTRIBUTES))) { + SQL_RESV_LOG(WARN, "failed to add member to bitset!", K(ret)); + } + } + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("index option should not specify kv attributes", K(ret)); + } + break; + } default: { /* won't be here */ ret = OB_ERR_UNEXPECTED; @@ -4238,6 +4319,8 @@ void ObDDLResolver::reset() { tablespace_id_ = OB_INVALID_ID; table_dop_ = DEFAULT_TABLE_DOP; hash_subpart_num_ = -1; + ttl_definition_.reset(); + kv_attributes_.reset(); } bool ObDDLResolver::is_valid_prefix_key_type(const ObObjTypeClass column_type_class) @@ -10948,6 +11031,32 @@ int ObDDLResolver::deep_copy_string_in_part_expr(ObPartitionedStmt* stmt) return ret; } +int ObDDLResolver::get_ttl_columns(const ObString &ttl_definition, ObIArray &ttl_columns) +{ + int ret = OB_SUCCESS; + if (ttl_definition.empty()) { + // do nothing + } else { + ObString right = ttl_definition; + bool is_end = false; + while (OB_SUCC(ret) && !is_end) { + ObString left = right.split_on(','); + if (left.empty()) { + left = right; + is_end = true; + } + ObString column_name = left.split_on('+').trim(); + if (column_name.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null column name", K(ret)); + } else if (OB_FAIL(ttl_columns.push_back(column_name))) { + LOG_WARN("fail to add column name", K(ret), K(column_name)); + } + } + } + return ret; +} + int ObDDLResolver::deep_copy_column_expr_name(common::ObIAllocator &allocator, ObIArray &exprs) { @@ -10972,5 +11081,54 @@ int ObDDLResolver::deep_copy_column_expr_name(common::ObIAllocator &allocator, return ret; } +int ObDDLResolver::check_ttl_definition(const ParseNode *node) +{ + int ret = OB_SUCCESS; + const ObColumnSchemaV2 *column_schema = NULL; + const ObTableSchema *tbl_schema = NULL; + if (OB_ISNULL(schema_checker_) || OB_ISNULL(stmt_) || + OB_ISNULL(session_info_) || OB_ISNULL(node) || node->type_ != T_TTL_DEFINITION) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(ERROR,"unexpected null value", K(ret), K_(schema_checker), + K_(stmt), K_(session_info), K(node)); + } else if (node->type_ != T_TTL_DEFINITION) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(ERROR,"unexpected node type", K(ret), K(node->type_)); + } else if (stmt::T_CREATE_TABLE == stmt_->get_stmt_type()) { + ObCreateTableStmt *create_table_stmt = static_cast(stmt_); + tbl_schema = &create_table_stmt->get_create_table_arg().schema_; + } else if (stmt::T_ALTER_TABLE == stmt_->get_stmt_type()) { + ObAlterTableStmt *alter_table_stmt = static_cast(stmt_); + if (OB_FAIL(schema_checker_->get_table_schema(session_info_->get_effective_tenant_id(), + database_name_, table_name_, false, tbl_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(session_info_->get_effective_tenant_id()), + K(alter_table_stmt->get_alter_table_arg())); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported statement for TTL expression", K(ret), K(stmt_->get_stmt_type())); + } + + if (OB_SUCC(ret)) { + ObString ttl_definition(node->str_len_, node->str_value_); + ObSEArray ttl_columns; + if (OB_FAIL(get_ttl_columns(ttl_definition, ttl_columns))) { + LOG_WARN("fail to get ttl columns", K(ttl_definition)); + } + for (int i = 0; i < OB_SUCC(ret) && ttl_columns.count(); i++) { + ObString column_name = ttl_columns.at(i); + if (NULL == (column_schema = tbl_schema->get_column_schema(column_name))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl column is invalid", K(ret)); + } else if ((!ob_is_datetime_tc(column_schema->get_data_type()))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("invalid ttl expression, ttl column type should be datetime or timestamp", K(ret), K(column_schema->get_data_type())); + } + } + } + + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 6bb73ab640..fa0d9424ba 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -849,6 +849,10 @@ protected: int deep_copy_string_in_part_expr(ObPartitionedStmt* stmt); int deep_copy_column_expr_name(common::ObIAllocator &allocator, ObIArray &exprs); + int check_ttl_definition(const ParseNode *node); + + int get_ttl_columns(const ObString &ttl_definition, ObIArray &ttl_columns); + void reset(); int64_t block_size_; int64_t consistency_level_; @@ -906,6 +910,8 @@ protected: int64_t table_dop_; // default value is 1 int64_t hash_subpart_num_; bool is_external_table_; + common::ObString ttl_definition_; + common::ObString kv_attributes_; private: template DISALLOW_COPY_AND_ASSIGN(ObDDLResolver); diff --git a/src/sql/resolver/expr/ob_raw_expr_util.h b/src/sql/resolver/expr/ob_raw_expr_util.h index 959eb11b43..2e10f25e3e 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.h +++ b/src/sql/resolver/expr/ob_raw_expr_util.h @@ -1141,7 +1141,6 @@ public: static bool is_column_ref_skip_implicit_cast(const ObRawExpr *expr); private : - static int create_real_cast_expr(ObRawExprFactory &expr_factory, ObRawExpr *src_expr, const ObExprResType &dst_type, diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index af6f50a83e..6949650736 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -1136,6 +1136,10 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(CheckpointSlog); break; } + case T_TABLE_TTL: { + REGISTER_STMT_RESOLVER(TableTTL); + break; + } default: { ret = OB_NOT_SUPPORTED; const char *type_name = get_type_name(parse_tree.type_); diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index f9816995ad..560705323b 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -65,6 +65,8 @@ #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" #include "storage/tablet/ob_tablet_multi_source_data.h" #include "storage/high_availability/ob_rebuild_service.h" +#include "observer/table/ttl/ob_ttl_service.h" +#include "observer/table/ttl/ob_tenant_tablet_ttl_mgr.h" #include "share/wr/ob_wr_service.h" namespace oceanbase @@ -309,6 +311,21 @@ int ObLS::init(const share::ObLSID &ls_id, LOG_INFO("heartbeat service is registered successfully"); } + if (OB_SUCC(ret) && is_user_tenant(tenant_id)) { + if (ls_id.is_sys_ls()) { + REGISTER_TO_LOGSERVICE(logservice::TTL_LOG_BASE_TYPE, MTL(table::ObTTLService *)); + LOG_INFO("register tenant ttl service complete", KR(ret)); + } else if (ls_id.is_user_ls()) { + if (OB_FAIL(tablet_ttl_mgr_.init(this))) { + LOG_WARN("fail to init tablet ttl manager", KR(ret)); + } else { + REGISTER_TO_LOGSERVICE(logservice::TTL_LOG_BASE_TYPE, &tablet_ttl_mgr_); + LOG_INFO("register tenant tablet ttl mgr complete", KR(ret)); + } + } + } + + if (OB_SUCC(ret)) { // don't delete it election_priority_.set_ls_id(ls_id); is_inited_ = true; @@ -846,6 +863,15 @@ void ObLS::destroy() GCTX.wr_service_); } + if (is_user_tenant(MTL_ID())) { + if (ls_meta_.ls_id_.is_sys_ls()) { + UNREGISTER_FROM_LOGSERVICE(logservice::TTL_LOG_BASE_TYPE, MTL(table::ObTTLService *)); + } else if (ls_meta_.ls_id_.is_user_ls()) { + UNREGISTER_FROM_LOGSERVICE(logservice::TTL_LOG_BASE_TYPE, tablet_ttl_mgr_); + tablet_ttl_mgr_.destroy(); + } + } + tx_table_.destroy(); lock_table_.destroy(); ls_tablet_svr_.destroy(); diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 039e6055a9..3dfd872e05 100755 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -66,6 +66,7 @@ #include "storage/high_availability/ob_ls_member_list_service.h" #include "storage/high_availability/ob_ls_block_tx_service.h" #include "storage/high_availability/ob_ls_transfer_info.h" +#include "observer/table/ttl/ob_tenant_tablet_ttl_mgr.h" namespace oceanbase { @@ -909,6 +910,7 @@ private: rootserver::ObLSRecoveryStatHandler ls_recovery_stat_handler_; ObLSMemberListService member_list_service_; ObLSBlockTxService block_tx_service_; + table::ObTenantTabletTTLMgr tablet_ttl_mgr_; private: bool is_inited_; uint64_t tenant_id_; diff --git a/src/storage/memtable/ob_lock_wait_mgr.cpp b/src/storage/memtable/ob_lock_wait_mgr.cpp index 2557399b47..bd57418a13 100644 --- a/src/storage/memtable/ob_lock_wait_mgr.cpp +++ b/src/storage/memtable/ob_lock_wait_mgr.cpp @@ -482,15 +482,17 @@ ObLink* ObLockWaitMgr::check_timeout() iter->on_retry_lock(hash); TRANS_LOG(INFO, "current task should be waken up cause reaching run ts", K(*iter)); } else if (0 == iter->sessid_) { - //do nothing, may be rpc plan, sessionid is not setted + // do nothing, may be rpc plan, sessionid is not setted } else if (NULL != deadlocked_session && is_deadlocked_session_(deadlocked_session, iter->sessid_)) { node2del = iter; TRANS_LOG(INFO, "session is deadlocked, pop the request", "sessid", iter->sessid_, K(*iter)); + } else { + // do nothing } - if (need_check_session) { + if (need_check_session && iter->sessid_ != 0) { // when lock wait in dag worker, session is not exist sql::ObSQLSessionInfo *session_info = NULL; int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index b07018fc5b..8c2764203d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -75,6 +75,7 @@ enable_async_syslog enable_cgroup enable_ddl enable_early_lock_release +enable_kv_ttl enable_major_freeze enable_monotonic_weak_read enable_ob_ratelimit @@ -106,6 +107,8 @@ ignore_replay_checksum_error index_block_cache_priority internal_sql_execute_timeout job_queue_processes +kv_ttl_duty_duration +kv_ttl_history_recycle_interval large_query_threshold large_query_worker_percentage leak_mod_to_check diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index 684ca867eb..428c0b1983 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -213,6 +213,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 407 __all_mock_fk_parent_table_column 0 201001 1 408 __all_mock_fk_parent_table_column_history 0 201001 1 409 __all_log_restore_source 0 201001 1 +410 __all_kv_ttl_task 0 201001 1 +411 __all_kv_ttl_task_history 0 201001 1 412 __all_service_epoch 0 201001 1 413 __all_spatial_reference_systems 0 201001 1 416 __all_column_checksum_error_info 0 201001 1 @@ -576,6 +578,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12323 __all_virtual_mock_fk_parent_table_column_history 2 201001 1 12324 __all_virtual_log_restore_source 2 201001 1 12325 __all_virtual_query_response_time 2 201001 1 +12326 __all_virtual_kv_ttl_task 2 201001 1 +12327 __all_virtual_kv_ttl_task_history 2 201001 1 12330 __all_virtual_column_checksum_error_info 2 201001 1 12331 __all_virtual_kvcache_handle_leak_info 2 201001 1 12334 __all_virtual_tablet_compaction_info 2 201001 1 @@ -876,11 +880,15 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21297 DBA_OB_DEADLOCK_EVENT_HISTORY 1 201001 1 21298 CDB_OB_DEADLOCK_EVENT_HISTORY 1 201001 1 21299 CDB_OB_SYS_VARIABLES 1 201001 1 +21300 DBA_OB_KV_TTL_TASKS 1 201001 1 +21301 DBA_OB_KV_TTL_TASK_HISTORY 1 201001 1 21302 GV$OB_LOG_STAT 1 201001 1 21303 V$OB_LOG_STAT 1 201001 1 21304 ST_GEOMETRY_COLUMNS 1 201002 1 21305 ST_SPATIAL_REFERENCE_SYSTEMS 1 201002 1 21306 QUERY_RESPONSE_TIME 1 201002 1 +21307 CDB_OB_KV_TTL_TASKS 1 201001 1 +21308 CDB_OB_KV_TTL_TASK_HISTORY 1 201001 1 21311 DBA_RSRC_PLANS 1 201001 1 21312 DBA_RSRC_PLAN_DIRECTIVES 1 201001 1 21313 DBA_RSRC_GROUP_MAPPINGS 1 201001 1 diff --git a/unittest/observer/table/test_create_executor.cpp b/unittest/observer/table/test_create_executor.cpp index f4467f5f37..2594815c70 100644 --- a/unittest/observer/table/test_create_executor.cpp +++ b/unittest/observer/table/test_create_executor.cpp @@ -166,6 +166,7 @@ void TestCreateExecutor::fake_ctx_init_common(ObTableCtx &fake_ctx, ObTableSchem g_sess_node_val.is_inited_ = true; g_sess_node_val.sess_info_.test_init(0, 0, 0, NULL); g_sess_node_val.sess_info_.load_all_sys_vars(schema_guard_); + fake_ctx.init_phy_plan_ctx(); } TEST_F(TestCreateExecutor, scan) @@ -531,7 +532,12 @@ TEST_F(TestCreateExecutor, test_cache) // get lib cache ObPlanCache *lib_cache = &plan_cache; // construct cache key - ObTableApiCacheKey cache_key(1001, 1001, 1, ObTableOperationType::Type::INSERT); + ObTableApiCacheKey cache_key; + cache_key.table_id_ = 1001; + cache_key.index_table_id_ = 1001; + cache_key.schema_version_ = 1; + cache_key.operation_type_ = ObTableOperationType::Type::INSERT; + cache_key.is_ttl_table_ = false; // construct ctx ObILibCacheCtx ctx; // construct cache obj diff --git a/unittest/observer/table/test_hfilter_parser.cpp b/unittest/observer/table/test_hfilter_parser.cpp index 6bbcddaa75..a6c8c529e7 100644 --- a/unittest/observer/table/test_hfilter_parser.cpp +++ b/unittest/observer/table/test_hfilter_parser.cpp @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 OceanBase + * Copyright (c) 2022 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: