[FEAT MERGE] column store ddl

Co-authored-by: AnimationFan <30674773338@qq.com>
Co-authored-by: simonjoylet <simonjoylet@gmail.com>
Co-authored-by: Monk-Liu <1152761042@qq.com>

deps/oblib/src/common/object/ob_object.cpp

@@ -571,7 +571,7 @@ int ObLobLocatorV2::get_disk_locator(ObString &disc_loc_buff) const
     int64_t handle_size = reinterpret_cast<intptr_t>(disk_loc) - reinterpret_cast<intptr_t>(ptr_);
     if (handle_size > size_) {
       ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("get invalid handle size", K(ret), K(size_), K(disk_loc), K(ptr_));
+      LOG_WARN("get invalid handle size", K(ret), K(size_), K(disk_loc), K(ptr_), K(handle_size));
     } else {
       if (disk_loc->in_row_) {
         handle_size = size_ - handle_size;

deps/oblib/src/common/object/ob_object.h

@@ -501,6 +501,7 @@ struct ObLobId
   bool operator >(const ObLobId &other) const;
   TO_STRING_KV(K_(tablet_id), K_(lob_id));
   void reset();
+  inline bool is_valid() const {return tablet_id_ != 0 && lob_id_ != 0;}
   uint64_t tablet_id_;
   uint64_t lob_id_;
 };

deps/oblib/src/lib/allocator/ob_block_alloc_mgr.h

@@ -37,7 +37,7 @@ public:
     if (used_after_alloc > limit_) {
       ATOMIC_AAF(&hold_, -size);
       if (REACH_TIME_INTERVAL(1000 * 1000)) {
-        _OB_LOG_RET(WARN, common::OB_ERR_UNEXPECTED, "block alloc over limit, limit=%ld alloc_size=%ld", limit_, size);
+        _OB_LOG_RET(WARN, common::OB_ALLOCATE_MEMORY_FAILED, "block alloc over limit, limit=%ld alloc_size=%ld", limit_, size);
       }
     } else if (NULL == (ret = (void*)ob_malloc(size, attr))) {
       ATOMIC_AAF(&hold_, -size);

deps/oblib/src/lib/hash/ob_linear_hash_map.h

@@ -394,9 +394,9 @@ public:
   // m_seg_sz and s_seg_sz are the sizes of micro-segment and standard-segment.
   // Set m_seg_sz = 0 to disable micro-segment.
   // dir_init_sz is the initial size of directory, it doubles when overflows.
+  int init(const ObMemAttr &mem_attr);
   int init(const lib::ObLabel &label = LABEL, uint64_t tenant_id = TENANT_ID);
-  int init(uint64_t m_seg_sz, uint64_t s_seg_sz, uint64_t dir_init_sz,
-      const lib::ObLabel &label = LABEL, uint64_t tenant_id = TENANT_ID);
+  int init(uint64_t m_seg_sz, uint64_t s_seg_sz, uint64_t dir_init_sz, const ObMemAttr &mem_attr);
   int destroy();
   // Load factor control.
   int set_load_factor_lmt(double lower_lmt, double upper_lmt);
@@ -867,34 +867,38 @@ void ObLinearHashMap<Key, Value, MemMgrTag>::BlurredIterator::rewind()
 
 // Public functions.
 template <typename Key, typename Value, typename MemMgrTag>
-int ObLinearHashMap<Key, Value, MemMgrTag>::init(const lib::ObLabel &label /*= LABEL*/,
-                                                 uint64_t tenant_id /*=TENANT_ID*/)
+int ObLinearHashMap<Key, Value, MemMgrTag>::init(const ObMemAttr &mem_attr)
 {
   return init(OB_MALLOC_NORMAL_BLOCK_SIZE, /* Small segment. */
               OB_MALLOC_BIG_BLOCK_SIZE, /* Large segment. */
               DIR_SZ_L_LMT, /* Dir size, small when init, expand * 2. */
-              label,
-              tenant_id);
+              mem_attr);
+}
+
+template <typename Key, typename Value, typename MemMgrTag>
+int ObLinearHashMap<Key, Value, MemMgrTag>::init(const lib::ObLabel &label /*= LABEL*/,
+                                                 uint64_t tenant_id /*=TENANT_ID*/)
+{
+  return init(ObMemAttr(tenant_id, label));
 }
 
 template <typename Key, typename Value, typename MemMgrTag>
 int ObLinearHashMap<Key, Value, MemMgrTag>::init(uint64_t m_seg_sz, uint64_t s_seg_sz, uint64_t dir_init_sz,
-    const lib::ObLabel &label /*= LABEL*/, uint64_t tenant_id /*=TENANT_ID*/)
+    const ObMemAttr &mem_attr)
 {
   const double LOAD_FCT_DEF_U_LMT = 1;
   const double LOAD_FCT_DEF_L_LMT = 0.01;
 
   int ret = OB_SUCCESS;
   /* Memory alloc from MemMgr, and its static, so label and tenant_id no longer used. */
-  memattr_.tenant_id_ = tenant_id;
-  memattr_.label_ = label;
+  memattr_ = mem_attr;
   load_factor_u_limit_ = LOAD_FCT_DEF_U_LMT;
   load_factor_l_limit_ = LOAD_FCT_DEF_L_LMT;
   load_factor_ = 0.0;
   set_Lp_(0, 0);
   init_haz_();
   init_foreach_();
-  if (OB_SUCCESS != (ret = mem_mgr_.init(tenant_id)))
+  if (OB_SUCCESS != (ret = mem_mgr_.init(mem_attr.tenant_id_)))
   { }
   else if (OB_SUCCESS != (ret = init_d_arr_(m_seg_sz, s_seg_sz, dir_init_sz)))
   { }

deps/oblib/src/lib/ob_define.h

@@ -826,8 +826,10 @@ const int64_t MAX_COLUMN_YES_NO_LENGTH = 3;
 const int64_t MAX_COLUMN_VARCHAR_LENGTH = 262143;
 const int64_t MAX_COLUMN_CHAR_LENGTH = 255;
 //column group
+const uint64_t INVALID_COLUMN_GROUP_ID = 0;
 const uint64_t DEFAULT_TYPE_COLUMN_GROUP_ID = 1; // reserve 2~999
 const uint64_t COLUMN_GROUP_START_ID = 1000;
+const uint64_t DEFAULT_CUSTOMIZED_CG_NUM = 2;
 
 //Oracle
 const int64_t MAX_ORACLE_COMMENT_LENGTH = 4000;

deps/oblib/unittest/lib/hash/test_linear_hash_map.cpp

@@ -150,7 +150,7 @@ TEST(ObLinearHashMap, EStest1)
 
   // Init map.
   Map map;
-  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz));
+  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz, ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_LINEAR_HASH_MAP)));
   // Validate some settings.
   EXPECT_EQ(m_sz / sizeof(Map::Bucket), map.L0_bkt_n_);
   {
@@ -217,7 +217,7 @@ TEST(ObLinearHashMap, ACCSStest1)
   uint64_t m_sz = 1 << 12; // 4KB
   uint64_t s_sz = 1 << 16; // 64KB
   Map map;
-  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz));
+  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz, ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_LINEAR_HASH_MAP)));
   map.load_factor_l_limit_ = 0;
   map.load_factor_u_limit_ = 100000;
   const int64_t limit = 10000;
@@ -259,7 +259,7 @@ TEST(ObLinearHashMap, ACCSStest2)
   uint64_t m_sz = 1 << 12; // 4KB
   uint64_t s_sz = 1 << 16; // 64KB
   const uint64_t maxL = 6; // will use both m-seg and s-seg
-  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz));
+  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz, ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_LINEAR_HASH_MAP)));
   map.load_factor_l_limit_ = 0;
   map.load_factor_u_limit_ = 100000000;
   const int64_t key_n = (int64_t)2 << maxL; // more than 2 keys in bucket when it reaches maxL.
@@ -335,7 +335,7 @@ TEST(ObLinearHashMap, ACCSStest3)
   const int64_t thread_n = 4;
   pthread_t threads[thread_n];
   Map map;
-  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz));
+  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz, ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_LINEAR_HASH_MAP)));
   map.load_factor_l_limit_ = 0;
   map.load_factor_u_limit_ = 100000;
   // Modify the dir size to an extremely short size.
@@ -392,7 +392,7 @@ TEST(ObLinearHashMap, ACCSStest4)
   Map map;
   uint64_t m_sz = 1 << 12; // 4KB
   uint64_t s_sz = 1 << 16; // 64KB
-  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz));
+  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz, ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_LINEAR_HASH_MAP)));
   map.load_factor_l_limit_ = 0;
   map.load_factor_u_limit_ = 100000;
   uint64_t L, p;
@@ -516,7 +516,7 @@ TEST(ObLinearHashMap, ACCSStest5)
   const int64_t thread_n = 4;
   pthread_t threads[thread_n];
   Map map;
-  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz));
+  EXPECT_EQ(OB_SUCCESS, map.init(m_sz, s_sz, m_sz, ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_LINEAR_HASH_MAP)));
   map.load_factor_l_limit_ = 0;
   map.load_factor_u_limit_ = 100000;
   // Insert.

mittest/mtlenv/mock_tenant_module_env.h

@@ -707,6 +707,7 @@ int MockTenantModuleEnv::init()
       MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy);
       MTL_BIND2(mtl_new_default, ObTenantCGReadInfoMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
       MTL_BIND2(mtl_new_default, ObDecodeResourcePool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
+      MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
       MTL_BIND2(mtl_new_default, ObEmptyReadBucket::mtl_init, nullptr, nullptr, nullptr, ObEmptyReadBucket::mtl_destroy);
       MTL_BIND2(mtl_new_default, ObRebuildService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
       MTL_BIND2(mtl_new_default, table::ObHTableLockMgr::mtl_init, nullptr, nullptr, nullptr, table::ObHTableLockMgr::mtl_destroy);

mittest/mtlenv/storage/blocksstable/CMakeLists.txt

@@ -20,3 +20,6 @@ storage_dml_unittest(test_co_sstable_row_scanner)
 storage_dml_unittest(test_pushdown_aggregate)
 storage_dml_unittest(test_cg_group_by_scanner)
 storage_dml_unittest(test_cs_cg_group_by_scanner)
+storage_dml_unittest(test_ddl_merge_row_scanner)
+storage_dml_unittest(test_ddl_merge_row_multi_scanner)
+storage_dml_unittest(test_direct_load)

mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h

@@ -68,17 +68,22 @@ public:
   virtual void TearDown();
   virtual void prepare_schema();
   virtual void prepare_data(const int64_t micro_block_size = 0);
+  virtual void prepare_partial_ddl_data();
   virtual void prepare_cg_data();
   virtual void insert_data(ObMacroBlockWriter &data_writer); // override to define data in sstable
   virtual void insert_cg_data(ObMacroBlockWriter &data_writer); // override to define data in sstable
+  virtual void insert_partial_data(ObMacroBlockWriter &data_writer, const int64_t row_cnt); // override to define data in partial_sstable
   static void convert_to_multi_version_row(const ObDatumRow &org_row, const ObTableSchema &schema,
       const int64_t snapshot_version, const ObDmlFlag dml_flag, ObDatumRow &multi_row);
   static void fake_freeze_info();
   virtual ObITable::TableType get_merged_table_type() const;
   void prepare_query_param(const bool is_reverse_scan, ObArenaAllocator *allocator = nullptr);
   void destroy_query_param();
+  void prepare_contrastive_sstable();
   void prepare_ddl_kv();
+  void prepare_merge_ddl_kvs();
   void close_builder_and_prepare_sstable(const int64_t column_cnt);
+  void prepare_partial_sstable(const int64_t column_cnt);
   int gen_create_tablet_arg(const int64_t tenant_id,
     const share::ObLSID &ls_id,
     const ObTabletID &tablet_id,
@@ -96,16 +101,24 @@ protected:
   static const uint64_t tablet_id_ = 200001;
   static const uint64_t TEST_TABLE_ID = 200001;
   static const uint64_t ls_id_ = 1001;
+  static const int64_t DDL_KVS_CNT = 3;
   ObMergeType merge_type_;
   int64_t max_row_cnt_;
+  int64_t max_partial_row_cnt_;
+  int64_t partial_kv_start_idx_;
   ObTableSchema table_schema_;
   ObTableSchema index_schema_;
   ObRowGenerate row_generate_;
   int64_t row_cnt_;
+  int64_t partial_sstable_row_cnt_;
+  ObSSTable partial_sstable_;
   ObSSTable sstable_;
-  storage::ObDDLKV ddl_kv_;
+  storage::ObDDLMemtable ddl_kv_;
+  storage::ObDDLKVHandle ddl_kvs_;
   ObSSTableIndexBuilder *root_index_builder_;
+  ObSSTableIndexBuilder *merge_root_index_builder_;
   ObMicroBlockData root_block_data_buf_;
+  ObMicroBlockData merge_root_block_data_buf_;
   ObRowStoreType row_store_type_;
   int64_t max_row_seed_;
   int64_t min_row_seed_;
@@ -126,6 +139,7 @@ protected:
   int64_t rows_per_mirco_block_;
   int64_t mirco_blocks_per_macro_block_;
   bool is_cg_data_;
+  bool is_ddl_merge_data_;
 };
 
 ObArenaAllocator TestIndexBlockDataPrepare::allocator_;
@@ -153,6 +167,7 @@ void TestIndexBlockDataPrepare::prepare_query_param(const bool is_reverse_scan,
   }
   context_.store_ctx_ = &store_ctx_;
   context_.ls_id_ = ls_id_;
+  context_.tablet_id_ = tablet_id_;
   context_.allocator_ = test_allocator;
   context_.stmt_allocator_ = test_allocator;
   context_.limit_param_ = nullptr;
@@ -180,7 +195,9 @@ TestIndexBlockDataPrepare::TestIndexBlockDataPrepare(
   : merge_type_(merge_type),
     max_row_cnt_(max_row_cnt),
     row_cnt_(0),
+    partial_sstable_row_cnt_(0),
     root_index_builder_(nullptr),
+    merge_root_index_builder_(nullptr),
     row_store_type_(row_store_type),
     max_row_seed_(0),
     min_row_seed_(0),
@@ -191,7 +208,8 @@ TestIndexBlockDataPrepare::TestIndexBlockDataPrepare(
     need_agg_data_(need_aggregate_data),
     rows_per_mirco_block_(rows_per_mirco_block),
     mirco_blocks_per_macro_block_(mirco_blocks_per_macro_block),
-    is_cg_data_(false)
+    is_cg_data_(false),
+    is_ddl_merge_data_(false)
 {
 }
 
@@ -283,9 +301,12 @@ void TestIndexBlockDataPrepare::SetUp()
   ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema_, allocator_));
   ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
   sstable_.key_.table_type_ = ObITable::TableType::COLUMN_ORIENTED_SSTABLE;
+  partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
 
   if (is_cg_data_) {
     prepare_cg_data();
+  } else if (is_ddl_merge_data_) {
+    prepare_partial_ddl_data();
   } else {
     prepare_data();
   }
@@ -293,17 +314,32 @@ void TestIndexBlockDataPrepare::SetUp()
 void TestIndexBlockDataPrepare::TearDown()
 {
   sstable_.reset();
+  partial_sstable_.reset();
   ddl_kv_.reset();
+  ddl_kvs_.reset();
   cg_read_info_handle_.reset();
   if (nullptr != root_block_data_buf_.buf_) {
     allocator_.free((void *)root_block_data_buf_.buf_);
     root_block_data_buf_.buf_ = nullptr;
   }
+  if (nullptr != root_block_data_buf_.buf_) {
+    allocator_.free((void *)root_block_data_buf_.buf_);
+    root_block_data_buf_.buf_ = nullptr;
+  }
+  if (nullptr != merge_root_block_data_buf_.buf_) {
+    allocator_.free((void *)merge_root_block_data_buf_.buf_);
+    merge_root_block_data_buf_.buf_ = nullptr;
+  }
   if (nullptr != root_index_builder_) {
     root_index_builder_->~ObSSTableIndexBuilder();
     allocator_.free((void *)root_index_builder_);
     root_index_builder_ = nullptr;
   }
+  if (nullptr != merge_root_index_builder_) {
+    merge_root_index_builder_->~ObSSTableIndexBuilder();
+    allocator_.free((void *)merge_root_index_builder_);
+    merge_root_index_builder_ = nullptr;
+  }
   ObLSID ls_id(ls_id_);
   ObTabletID tablet_id(tablet_id_);
   ObLSHandle ls_handle;
@@ -345,7 +381,7 @@ ObITable::TableType TestIndexBlockDataPrepare::get_merged_table_type() const
   } else if (META_MAJOR_MERGE == merge_type_) {
     table_type = ObITable::TableType::META_MAJOR_SSTABLE;
   } else if (DDL_KV_MERGE == merge_type_) {
-    table_type = ObITable::TableType::DDL_DUMP_SSTABLE;
+    table_type = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
   } else { // MINOR_MERGE || HISTORY_MINOR_MERGE
     table_type = ObITable::TableType::MINOR_SSTABLE;
   }
@@ -354,7 +390,7 @@ ObITable::TableType TestIndexBlockDataPrepare::get_merged_table_type() const
 
 void TestIndexBlockDataPrepare::prepare_schema()
 {
-    ObColumnSchemaV2 column;
+  ObColumnSchemaV2 column;
   //init table schema
   uint64_t table_id =  TEST_TABLE_ID;
   table_schema_.reset();
@@ -502,10 +538,9 @@ void TestIndexBlockDataPrepare::close_builder_and_prepare_sstable(const int64_t
   row_store_type_ = root_row_store_type;
 
   ObITable::TableKey table_key;
-  int64_t table_id = 3001;
   int64_t tenant_id = 1;
   table_key.table_type_ = get_merged_table_type();
-  table_key.tablet_id_ = table_id;
+  table_key.tablet_id_ = tablet_id_;
   table_key.version_range_.snapshot_version_ = SNAPSHOT_VERSION;
 
   ObTabletCreateSSTableParam param;
@@ -685,7 +720,7 @@ void TestIndexBlockDataPrepare::prepare_ddl_kv()
 
   share::SCN ddl_start_scn;
   ddl_start_scn.convert_from_ts(ObTimeUtility::current_time());
-  ASSERT_EQ(OB_SUCCESS, ddl_kv_.init(*tablet_handle.get_obj(), ddl_start_scn, sstable_.get_data_version(), ddl_start_scn, 4000));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_.init(*tablet_handle.get_obj(), sstable_.get_key(), ddl_start_scn, DATA_CURRENT_VERSION));
 
   SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
     ObDatumRange query_range;
@@ -711,7 +746,6 @@ void TestIndexBlockDataPrepare::prepare_ddl_kv()
       }
     }
     ASSERT_EQ(OB_ITER_END, ret);
-    ASSERT_EQ(OB_SUCCESS, ddl_kv_.block_meta_tree_.build_sorted_rowkeys());
   }
 }
 
@@ -813,6 +847,326 @@ void TestIndexBlockDataPrepare::convert_to_multi_version_row(const ObDatumRow &o
   multi_row.mvcc_row_flag_.set_last_multi_version_row(true);
 }
 
+void TestIndexBlockDataPrepare::prepare_partial_ddl_data()
+{
+  prepare_contrastive_sstable();
+  ObMacroBlockWriter writer;
+  ObMacroDataSeq start_seq(0);
+  start_seq.set_data_block();
+  row_generate_.reset();
+  ObWholeDataStoreDesc desc(true/*is ddl*/);
+  share::SCN end_scn;
+  end_scn.convert_from_ts(ObTimeUtility::current_time());
+  ASSERT_EQ(OB_SUCCESS, desc.init(table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, end_scn));
+  void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
+  merge_root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder();
+  ASSERT_NE(nullptr, merge_root_index_builder_);
+  desc.get_desc().sstable_index_builder_ = merge_root_index_builder_;
+  ASSERT_TRUE(desc.is_valid());
+  if (need_agg_data_) {
+    ASSERT_EQ(OB_SUCCESS, desc.get_desc().col_desc_->agg_meta_array_.assign(agg_col_metas_));
+  }
+  ASSERT_EQ(OB_SUCCESS, merge_root_index_builder_->init(desc.get_desc()));
+  ASSERT_EQ(OB_SUCCESS, writer.open(desc.get_desc(), start_seq));
+  ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_, &allocator_));
+  const int64_t partial_row_cnt = max_partial_row_cnt_;
+  insert_partial_data(writer, partial_row_cnt);
+  ASSERT_EQ(OB_SUCCESS, writer.close());
+  // data write ctx has been moved to merge_root_index_builder_
+  ASSERT_EQ(writer.get_macro_block_write_ctx().get_macro_block_count(), 0);
+  data_macro_block_cnt_ = merge_root_index_builder_->roots_[0]->macro_metas_->count();
+  ASSERT_GE(data_macro_block_cnt_, 0);
+  int64_t column_cnt = 0;
+  ObTabletID tablet_id(TestIndexBlockDataPrepare::tablet_id_);
+  ObLSID ls_id(ls_id_);
+  ObLSHandle ls_handle;
+  ObTabletHandle tablet_handle;
+  ObLSService *ls_svr = MTL(ObLSService*);
+  ObStorageSchema *storage_schema = nullptr;
+  ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
+  ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
+  ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->load_storage_schema(allocator_, storage_schema));
+  ASSERT_EQ(OB_SUCCESS, storage_schema->get_stored_column_count_in_sstable(column_cnt));
+  prepare_partial_sstable(column_cnt);
+  prepare_merge_ddl_kvs();
+  ObTabletObjLoadHelper::free(allocator_, storage_schema);
+}
+
+void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt)
+{
+  ObSSTableMergeRes res;
+  OK(merge_root_index_builder_->close(res));
+  ObIndexTreeRootBlockDesc root_desc;
+  root_desc = res.root_desc_;
+  ASSERT_TRUE(root_desc.is_valid());
+  ObRowStoreType root_row_store_type = res.root_row_store_type_;
+  char *root_buf = nullptr;
+  int64_t root_size = 0;
+  if (root_desc.addr_.is_block()) {
+    // read macro block
+    ObMacroBlockReadInfo read_info;
+    ObMacroBlockHandle macro_handle;
+    const int64_t macro_block_size = 2 * 1024 * 1024;
+    ASSERT_EQ(OB_SUCCESS, root_desc.addr_.get_block_addr(read_info.macro_block_id_, read_info.offset_, read_info.size_));
+    read_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_DATA_READ);
+    read_info.offset_ = 0;
+    read_info.size_ = macro_block_size;
+    read_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS;
+    ASSERT_NE(nullptr, read_info.buf_ = reinterpret_cast<char*>(allocator_.alloc(read_info.size_)));
+    ASSERT_EQ(OB_SUCCESS, ObBlockManager::read_block(read_info, macro_handle));
+    ASSERT_NE(macro_handle.get_buffer(), nullptr);
+    ASSERT_EQ(macro_handle.get_data_size(), macro_block_size);
+    // get root block
+    int64_t block_offset = root_desc.addr_.offset_;
+    int64_t block_size = root_desc.addr_.size_;
+    const char *block_buf = macro_handle.get_buffer() + block_offset;
+    // decompress and decrypt root block
+    ObMicroBlockDesMeta meta(ObCompressorType::NONE_COMPRESSOR, root_row_store_type, 0, 0, nullptr);
+    ObMacroBlockReader reader;
+    const char *decomp_buf = nullptr;
+    int64_t decomp_size = 0;
+    bool is_compressed = false;
+    ASSERT_EQ(OB_SUCCESS, reader.decrypt_and_decompress_data(meta, block_buf, root_desc.addr_.size_,
+        decomp_buf, decomp_size, is_compressed, true, &allocator_));
+    root_buf = const_cast<char *>(decomp_buf);
+    root_size = decomp_size;
+  } else if (root_desc.is_mem_type()) {
+    root_buf = root_desc.buf_;
+    root_size = root_desc.addr_.size_;
+  } else {
+    STORAGE_LOG(INFO, "not supported root block", K(root_desc));
+    ASSERT_TRUE(false);
+  }
+  // deserialize micro block header in root block buf
+  ObMicroBlockHeader root_micro_header;
+  int64_t des_pos = 0;
+  ASSERT_EQ(OB_SUCCESS, root_micro_header.deserialize(root_buf, root_size, des_pos));
+  merge_root_block_data_buf_.buf_ = static_cast<char *>(allocator_.alloc(root_size));
+  merge_root_block_data_buf_.size_ = root_size;
+  int64_t copy_pos = 0;
+  ObMicroBlockHeader *copied_micro_header = nullptr;
+  ASSERT_EQ(OB_SUCCESS, root_micro_header.deep_copy(
+      (char *)merge_root_block_data_buf_.buf_, merge_root_block_data_buf_.size_, copy_pos, copied_micro_header));
+  ASSERT_TRUE(copied_micro_header->is_valid());
+  MEMCPY((char *)(merge_root_block_data_buf_.buf_ + copy_pos), root_buf + des_pos, root_size - des_pos);
+  row_store_type_ = root_row_store_type;
+  ObITable::TableKey table_key;
+  int64_t table_id = 3001;
+  int64_t tenant_id = 1;
+  table_key.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
+  table_key.tablet_id_ = table_id;
+  table_key.version_range_.snapshot_version_ = SNAPSHOT_VERSION;
+  ObTabletCreateSSTableParam param;
+  param.table_key_ = table_key;
+  param.schema_version_ = 10;
+  param.create_snapshot_version_ = 0;
+  param.progressive_merge_round_ = table_schema_.get_progressive_merge_round();
+  param.progressive_merge_step_ = 0;
+  param.table_mode_ = table_schema_.get_table_mode_struct();
+  param.index_type_ = table_schema_.get_index_type();
+  if (is_cg_data_) {
+    param.rowkey_column_cnt_ = 0;
+  } else {
+    param.rowkey_column_cnt_ = table_schema_.get_rowkey_column_num()
+      + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
+  }
+  ObSSTableMergeRes::fill_addr_and_data(res.root_desc_,
+    param.root_block_addr_, param.root_block_data_);
+  ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_,
+    param.data_block_macro_meta_addr_, param.data_block_macro_meta_);
+  param.is_meta_root_ = res.data_root_desc_.is_meta_root_;
+  param.max_merged_trans_version_ = res.max_merged_trans_version_;
+  param.row_count_ = res.row_count_;
+  param.root_row_store_type_ = root_row_store_type;
+  param.latest_row_store_type_ = table_schema_.get_row_store_type();
+  param.data_index_tree_height_ = root_desc.height_;
+  param.index_blocks_cnt_ = res.index_blocks_cnt_;
+  param.data_blocks_cnt_ = res.data_blocks_cnt_;
+  param.micro_block_cnt_ = res.micro_block_cnt_;
+  param.use_old_macro_block_count_ = res.use_old_macro_block_count_;
+  param.column_cnt_= column_cnt;
+  param.data_checksum_ = res.data_checksum_;
+  param.occupy_size_ = res.occupy_size_;
+  param.original_size_ = res.original_size_;
+  param.nested_offset_ = res.nested_offset_;
+  param.nested_size_ = res.nested_size_;
+  param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR;
+  param.data_block_ids_ = res.data_block_ids_;
+  param.other_block_ids_ = res.other_block_ids_;
+  param.ddl_scn_.convert_from_ts(ObTimeUtility::current_time());
+  param.filled_tx_scn_.set_min();
+  param.contain_uncommitted_row_ = false;
+  param.encrypt_id_ = res.encrypt_id_;
+  param.master_key_id_ = res.master_key_id_;
+  if (param.table_key_.is_co_sstable() && param.column_group_cnt_ <= 1) {
+    param.column_group_cnt_ = column_cnt + 2; /* set column group_cnt to avoid return err, cnt is calculated as each + all + default*/
+  }
+  MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
+  if (merge_type_ == MAJOR_MERGE) {
+    OK(ObSSTableMergeRes::fill_column_checksum_for_empty_major(param.column_cnt_, param.column_checksums_));
+  }
+  partial_sstable_.reset();
+  ASSERT_EQ(OB_SUCCESS, partial_sstable_.init(param, &allocator_));
+  STORAGE_LOG(INFO, "create partial_sstable param", K(param));
+}
+
+void TestIndexBlockDataPrepare::insert_partial_data(ObMacroBlockWriter &data_writer, const int64_t row_cnt)
+{
+  partial_sstable_row_cnt_ = 0;
+  int64_t seed = min_row_seed_;
+  ObDatumRow row;
+  ObDatumRow multi_row;
+  ASSERT_EQ(OB_SUCCESS, row.init(allocator_, MAX_TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, multi_row.init(allocator_, MAX_TEST_COLUMN_CNT));
+  ObDmlFlag flags[] = {DF_INSERT, DF_UPDATE, DF_DELETE};
+  int64_t rows_per_mirco_block = rows_per_mirco_block_;
+  int64_t rows_per_macro_block = rows_per_mirco_block_ * mirco_blocks_per_macro_block_;
+  int64_t rows_cnt = row_cnt;
+  if (INT64_MAX == rows_per_mirco_block_ || INT64_MAX == mirco_blocks_per_macro_block_) {
+    rows_per_mirco_block = INT64_MAX;
+    rows_per_macro_block = INT64_MAX;
+  }
+  while (true) {
+    if (partial_sstable_row_cnt_ >= rows_cnt) {
+      break;
+    }
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed, row));
+    ++seed;
+    ObDmlFlag dml = flags[partial_sstable_row_cnt_ % ARRAYSIZEOF(flags)]; // INSERT / UPDATE / DELETE
+    convert_to_multi_version_row(row, table_schema_, SNAPSHOT_VERSION, dml, multi_row);
+    ASSERT_EQ(OB_SUCCESS, data_writer.append_row(multi_row));
+    if (partial_sstable_row_cnt_ == 0) {
+      ObDatumRowkey &start_key = data_writer.last_key_;
+      ASSERT_EQ(OB_SUCCESS, start_key.deep_copy(start_key_, allocator_));
+    }
+    if (partial_sstable_row_cnt_ == rows_cnt - 1) {
+      ObDatumRowkey &end_key = data_writer.last_key_;
+      ASSERT_EQ(OB_SUCCESS, end_key.deep_copy(end_key_, allocator_));
+    }
+    if ((partial_sstable_row_cnt_ + 1) % rows_per_mirco_block == 0) {
+      OK(data_writer.build_micro_block());
+    }
+    if ((partial_sstable_row_cnt_ + 1) % rows_per_macro_block == 0) {
+      OK(data_writer.try_switch_macro_block());
+    }
+    ++partial_sstable_row_cnt_;
+  }
+  //max_row_seed_ = seed - 1;
+}
+
+void TestIndexBlockDataPrepare::prepare_contrastive_sstable()
+{
+  ObMacroBlockWriter writer;
+  ObMacroDataSeq start_seq(0);
+  start_seq.set_data_block();
+  row_generate_.reset();
+
+  ObWholeDataStoreDesc desc;
+  share::SCN end_scn;
+  end_scn.convert_from_ts(ObTimeUtility::current_time());
+  ASSERT_EQ(OB_SUCCESS, desc.init(table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, end_scn));
+  void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
+  root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder();
+  ASSERT_NE(nullptr, root_index_builder_);
+  desc.get_desc().sstable_index_builder_ = root_index_builder_;
+
+  ASSERT_TRUE(desc.is_valid());
+  if (need_agg_data_) {
+    ASSERT_EQ(OB_SUCCESS, desc.get_desc().col_desc_->agg_meta_array_.assign(agg_col_metas_));
+  }
+  ASSERT_EQ(OB_SUCCESS, root_index_builder_->init(desc.get_desc()));
+  ASSERT_EQ(OB_SUCCESS, writer.open(desc.get_desc(), start_seq));
+  ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_, &allocator_));
+
+  insert_data(writer);
+
+  ASSERT_EQ(OB_SUCCESS, writer.close());
+  // data write ctx has been moved to root_index_builder
+  ASSERT_EQ(writer.get_macro_block_write_ctx().get_macro_block_count(), 0);
+  data_macro_block_cnt_ = root_index_builder_->roots_[0]->macro_metas_->count();
+  ASSERT_GE(data_macro_block_cnt_, 0);
+
+  int64_t column_cnt = 0;
+  ObTabletID tablet_id(TestIndexBlockDataPrepare::tablet_id_);
+  ObLSID ls_id(ls_id_);
+  ObLSHandle ls_handle;
+  ObTabletHandle tablet_handle;
+  ObLSService *ls_svr = MTL(ObLSService*);
+  ObStorageSchema *storage_schema = nullptr;
+  ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
+  ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
+  ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->load_storage_schema(allocator_, storage_schema));
+  ASSERT_EQ(OB_SUCCESS, storage_schema->get_stored_column_count_in_sstable(column_cnt));
+  close_builder_and_prepare_sstable(column_cnt);
+  prepare_ddl_kv();
+  ObTabletObjLoadHelper::free(allocator_, storage_schema);
+}
+
+void TestIndexBlockDataPrepare::prepare_merge_ddl_kvs()
+{
+  ddl_kvs_.reset();
+  ObTabletHandle tablet_handle;
+  int ret = OB_SUCCESS;
+  ObLSID ls_id(ls_id_);
+  ObTabletID tablet_id(tablet_id_);
+  ObLSHandle ls_handle;
+  ObLSService *ls_svr = MTL(ObLSService*);
+  ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
+  ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
+  share::SCN ddl_start_scn;
+  ddl_start_scn.convert_from_ts(ObTimeUtility::current_time());
+  ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr *);
+  ASSERT_NE(t3m, nullptr);
+  ASSERT_EQ(OB_SUCCESS, t3m->acquire_ddl_kv(ddl_kvs_));
+  ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->init(ls_id, tablet_id, ddl_start_scn, sstable_.get_data_version(), ddl_start_scn, 4000));
+
+  ObITable::TableKey ddl_key = sstable_.get_key();
+  ddl_key.table_type_ = ObITable::TableType::MAJOR_SSTABLE;
+  for (int64_t i = 0; i < DDL_KVS_CNT; ++i) {
+    void *buf = allocator_.alloc(sizeof(ObDDLMemtable));
+    ASSERT_NE(nullptr, buf);
+    ObDDLMemtable *new_ddl_table = new (buf) ObDDLMemtable;
+    ASSERT_EQ(OB_SUCCESS, new_ddl_table->init(*tablet_handle.get_obj(), ddl_key, ddl_start_scn, 4000));
+    ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->get_ddl_memtables().push_back(new_ddl_table));
+  }
+  ObDDLKVHandle kv_handle;
+  ObDDLKvMgrHandle ddl_kv_mgr_handle;
+  ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, true /*CREATE*/));
+  ddl_kv_mgr_handle.get_obj()->set_ddl_kv(0, ddl_kvs_);
+  ddl_kv_mgr_handle.get_obj()->freeze_ddl_kv(ddl_start_scn, sstable_.get_data_version(), 4000, ddl_start_scn);
+  SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
+    ObDatumRange query_range;
+    query_range.set_whole_range();
+    ObDataMacroBlockMeta data_macro_meta;
+    ASSERT_EQ(OB_SUCCESS, meta_iter.open(query_range,
+                                         ObMacroBlockMetaType::DATA_BLOCK_META,
+                                         sstable_,
+                                         tablet_handle.get_obj()->get_rowkey_read_info(),
+                                         allocator_));
+    int64_t macro_idx = 0;
+    int64_t kv_idx = 0;
+    while (OB_SUCC(ret)) {
+      if (OB_FAIL(meta_iter.get_next(data_macro_meta))) {
+        if (OB_ITER_END != ret) {
+          STORAGE_LOG(WARN, "get data macro meta failed", K(ret));
+        }
+      } else {
+        STORAGE_LOG(INFO, "data_macro_meta_key", K(data_macro_meta));
+        ++macro_idx;
+        ObDDLMacroHandle macro_handle;
+        macro_handle.set_block_id(data_macro_meta.get_macro_id());
+        if (macro_idx > partial_kv_start_idx_) {
+          ObDataMacroBlockMeta *copied_meta = nullptr;
+          ASSERT_EQ(OB_SUCCESS, data_macro_meta.deep_copy(copied_meta, allocator_));
+          ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->get_ddl_memtables().at(kv_idx)->insert_block_meta_tree(macro_handle, copied_meta));
+          ++kv_idx;
+        }
+      }
+    }
+    ASSERT_EQ(OB_ITER_END, ret);
+  }
+}
+
 } // namespace blocksstable
 } // namespace oceanbase
 

mittest/mtlenv/storage/blocksstable/test_ddl_merge_row_multi_scanner.cpp

@@ -0,0 +1,772 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#define private public
+#define protected public
+
+#include "storage/access/ob_index_tree_prefetcher.h"
+#include "storage/access/ob_sstable_row_multi_scanner.h"
+#include "storage/access/ob_sstable_row_multi_getter.h"
+#include "ob_index_block_data_prepare.h"
+
+namespace oceanbase
+{
+using namespace storage;
+using namespace common;
+namespace blocksstable
+{
+class TestDDLMergeRowMultiScanner : public TestIndexBlockDataPrepare
+{
+public:
+  TestDDLMergeRowMultiScanner();
+  virtual ~TestDDLMergeRowMultiScanner();
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+
+  virtual void SetUp();
+  virtual void TearDown();
+protected:
+  void generate_range(const int64_t start, const int64_t end, ObDatumRange &range);
+public:
+  void test_one_case(
+      const ObIArray<int64_t> &start_seeds,
+      const int64_t count_per_range,
+      const bool is_reverse_scan);
+  void test_single_get_normal(const bool is_reverse_scan);
+  void test_single_get_border(const bool is_reverse_scan);
+  void test_multi_get_normal(const bool is_reverse_scan);
+  void test_multi_get_border(const bool is_reverse_scan);
+  void test_single_scan_normal(const bool is_reverse_scan);
+  void test_single_scan_border(const bool is_reverse_scan);
+  void test_multi_scan_multi_scan_range(
+      const bool is_reverse_scan,
+      const int64_t count_per_range);
+  void test_multi_scan_multi_get_with_scan(
+      const bool is_reverse_scan,
+      const int64_t count_per_range);
+
+protected:
+  static const int64_t TEST_MULTI_GET_CNT = 100;
+  enum CacheHitMode
+  {
+    HIT_ALL = 0,
+    HIT_NONE,
+    HIT_PART,
+    HIT_MAX,
+  };
+private:
+  ObArenaAllocator allocator_;
+  ObDatumRow start_row_;
+  ObDatumRow end_row_;
+};
+
+TestDDLMergeRowMultiScanner::TestDDLMergeRowMultiScanner()
+  : TestIndexBlockDataPrepare("Test sstable row multi scanner")
+{
+  is_ddl_merge_data_ = true;
+  max_row_cnt_ = 150000;
+  max_partial_row_cnt_ = 78881;
+  partial_kv_start_idx_ = 3;
+}
+
+TestDDLMergeRowMultiScanner::~TestDDLMergeRowMultiScanner()
+{
+
+}
+
+void TestDDLMergeRowMultiScanner::SetUpTestCase()
+{
+  TestIndexBlockDataPrepare::SetUpTestCase();
+}
+
+void TestDDLMergeRowMultiScanner::TearDownTestCase()
+{
+  TestIndexBlockDataPrepare::TearDownTestCase();
+}
+
+void TestDDLMergeRowMultiScanner::SetUp()
+{
+  TestIndexBlockDataPrepare::SetUp();
+  ObLSID ls_id(ls_id_);
+  ObTabletID tablet_id(tablet_id_);
+  ObLSHandle ls_handle;
+  ObLSService *ls_svr = MTL(ObLSService*);
+  ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
+  ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
+
+  ASSERT_EQ(OB_SUCCESS, start_row_.init(allocator_, TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, end_row_.init(allocator_, TEST_COLUMN_CNT));
+}
+
+void TestDDLMergeRowMultiScanner::TearDown()
+{
+  tablet_handle_.reset();
+  TestIndexBlockDataPrepare::TearDown();
+}
+
+void TestDDLMergeRowMultiScanner::generate_range(
+    const int64_t start,
+    const int64_t end,
+    ObDatumRange &range)
+{
+  ObDatumRowkey tmp_rowkey;
+  ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start, start_row_));
+  tmp_rowkey.assign(start_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+  ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.start_key_, allocator_));
+  ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(end, end_row_));
+  tmp_rowkey.assign(end_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+  ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.end_key_, allocator_));
+  range.border_flag_.set_inclusive_start();
+  range.border_flag_.set_inclusive_end();
+}
+
+void TestDDLMergeRowMultiScanner::test_one_case(
+    const ObIArray<int64_t> &start_seeds,
+    const int64_t count_per_range,
+    const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObDatumRange mscan_ranges[TEST_MULTI_GET_CNT];
+  ObSSTableRowMultiScanner<> scanner;
+  ObSSTableRowMultiScanner<> merge_ddl_scanner;
+  ObSEArray<ObDatumRange, TEST_MULTI_GET_CNT> ranges;
+  const ObDatumRow *prow = NULL;
+  const ObDatumRow *kv_prow = NULL;
+
+  ObDatumRow start_row;
+  ObDatumRow end_row;
+  ObDatumRow check_row;
+  ASSERT_EQ(OB_SUCCESS, start_row.init(allocator_, TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, end_row.init(allocator_, TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, check_row.init(allocator_, TEST_COLUMN_CNT));
+  for (int64_t i = 0; i < start_seeds.count(); ++i) {
+    ObDatumRowkey tmp_rowkey;
+    mscan_ranges[i].border_flag_.set_inclusive_start();
+    mscan_ranges[i].border_flag_.set_inclusive_end();
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start_seeds.at(i), start_row));
+    tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mscan_ranges[i].start_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start_seeds.at(i) + count_per_range - 1, end_row));
+    tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mscan_ranges[i].end_key_, allocator_));
+  }
+  for (int64_t i = 0; i < start_seeds.count(); ++i) {
+    ASSERT_EQ(OB_SUCCESS, ranges.push_back(mscan_ranges[i]));
+  }
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(iter_param_, context_, &sstable_, &ranges));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(iter_param_, context_, &partial_sstable_, &ranges));
+  for (int64_t i = 0; i < start_seeds.count(); ++i) {
+    for (int64_t j = 0; j < count_per_range; ++j) {
+      const int64_t k = is_reverse_scan ? start_seeds.at(i) + count_per_range - j - 1 : start_seeds.at(i) + j;
+      if (k < row_cnt_ || count_per_range == 1) {
+        ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(k, check_row));
+        if (k < row_cnt_) {
+          ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
+          ASSERT_TRUE(*prow == check_row);
+          ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_get_next_row(kv_prow));
+          ASSERT_TRUE(*kv_prow == check_row);
+        }
+      }
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  scanner.reuse();
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  merge_ddl_scanner.reuse();
+}
+
+void TestDDLMergeRowMultiScanner::test_single_get_normal(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObArray<int64_t> seeds;
+  // prepare query param and context
+  prepare_query_param(is_reverse_scan);
+
+  // row in first macro
+  ret = seeds.push_back(3);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // row in middle macro
+  seeds.reset();
+  seeds.push_back(row_cnt_ / 2);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // row in last macro, in cache
+  seeds.reset();
+  seeds.push_back(row_cnt_ - 3);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+  destroy_query_param();
+}
+
+void TestDDLMergeRowMultiScanner::test_single_get_border(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObArray<int64_t> seeds;
+  // prepare query param and context
+  prepare_query_param(is_reverse_scan);
+
+  // left border rowkey
+  ret = seeds.push_back(0);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // right border rowkey
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_ - 1);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // not exist
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+  destroy_query_param();
+}
+
+void TestDDLMergeRowMultiScanner::test_multi_get_normal(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObArray<int64_t> seeds;
+
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  // 2 rows exist
+  seeds.reuse();
+  for (int64_t i = 0; i < 2; ++i) {
+    ret = seeds.push_back(i * 11 + 2);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // TEST_MULTI_GET_CNT rows exist
+  seeds.reuse();
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ret = seeds.push_back(i);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // 2 row2 not exist
+  seeds.reuse();
+  for (int64_t i = 0; i < 2; ++i) {
+    ret = seeds.push_back(i * 11 + 2 + row_cnt_);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // TEST_MULTI_GET_CNT rows not exist
+  seeds.reuse();
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ret = seeds.push_back(i * 11 + 2 + row_cnt_);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // some row exist, while other rows not exist
+  seeds.reuse();
+  for (int64_t i = 0; i < 10; ++i) {
+    ret = seeds.push_back(i + (i % 2 ? row_cnt_ : 0));
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  destroy_query_param();
+}
+
+void TestDDLMergeRowMultiScanner::test_multi_get_border(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObArray<int64_t> seeds;
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  // first row of sstable
+  ret = seeds.push_back(0);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, false);
+
+  // last row of sstable
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_ - 1);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, false);
+
+  // 100 rows with same rowkey
+  seeds.reset();
+  for (int64_t i = 0; i < 100; ++i) {
+    ret = seeds.push_back(row_cnt_ / 2);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, 1, is_reverse_scan);
+  destroy_query_param();
+}
+
+void TestDDLMergeRowMultiScanner::test_single_scan_normal(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObArray<int64_t> seeds;
+  ObRandom random;
+  const int64_t random_start = random.get(0, 10000000) % row_cnt_;
+  const int64_t random_end = random.get(0, 100000000) % row_cnt_;
+  const int64_t start = std::min(random_start, random_end);
+  const int64_t end = std::max(random_start, random_end);
+
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  // multiple rows exist
+  ret = seeds.push_back(start);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, end - start, is_reverse_scan);
+
+  // multiple rows, partial exist
+  seeds.reset();
+  ret = seeds.push_back(start);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, row_cnt_ + 10 - start, is_reverse_scan);
+
+  // single row exist
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_ / 2);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // not exist
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_ + 10);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 10, is_reverse_scan);
+  destroy_query_param();
+}
+
+void TestDDLMergeRowMultiScanner::test_single_scan_border(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObArray<int64_t> seeds;
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  // full table scan
+  ret = seeds.push_back(0);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, row_cnt_, is_reverse_scan);
+
+  // first row of sstable
+  seeds.reset();
+  ret = seeds.push_back(0);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // last row of sstable
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_ - 1);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  // not exist
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+  destroy_query_param();
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_single_get_normal)
+{
+  bool is_reverse_scan = false;
+  test_single_get_normal(is_reverse_scan);
+  is_reverse_scan = true;
+  test_single_get_normal(is_reverse_scan);
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_single_get_border)
+{
+  bool is_reverse_scan = false;
+  test_single_get_border(is_reverse_scan);
+  is_reverse_scan = true;
+  test_single_get_border(is_reverse_scan);
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_multi_get_normal)
+{
+  bool is_reverse_scan = false;
+  test_multi_get_normal(is_reverse_scan);
+  is_reverse_scan = true;
+  test_multi_get_normal(is_reverse_scan);
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_multi_get_border)
+{
+  bool is_reverse_scan = false;
+  test_multi_get_border(is_reverse_scan);
+  is_reverse_scan = true;
+  test_multi_get_border(is_reverse_scan);
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_single_scan_normal)
+{
+  bool is_reverse_scan = false;
+  test_single_scan_normal(is_reverse_scan);
+  is_reverse_scan = true;
+  test_single_scan_normal(is_reverse_scan);
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_single_scan_border)
+{
+  bool is_reverse_scan = false;
+  test_single_scan_border(is_reverse_scan);
+  is_reverse_scan = true;
+  test_single_scan_border(is_reverse_scan);
+}
+
+void TestDDLMergeRowMultiScanner::test_multi_scan_multi_scan_range(
+    const bool is_reverse_scan,
+    const int64_t count_per_range)
+{
+  int ret = OB_SUCCESS;
+  ObStoreRange range;
+  ObArray<ObStoreRange> ranges;
+  ObStoreRow row;
+  ObStoreRowkey rowkey;
+  ObArray<int64_t> seeds;
+  ObSSTableRowMultiScanner<> scanner;
+
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  //left border rowkey
+  seeds.reset();
+  ret = seeds.push_back(0);
+  ASSERT_EQ(OB_SUCCESS, seeds.push_back(0));
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  //right border rowkey
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_ - 1);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  //not exist rowkey
+  seeds.reset();
+  ret = seeds.push_back(row_cnt_);
+  ASSERT_EQ(OB_SUCCESS, ret);
+  test_one_case(seeds, 1, is_reverse_scan);
+
+  //20 exist
+  seeds.reset();
+  for (int64_t i = 0; i < 50; ++i) {
+    ret = seeds.push_back(i);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, count_per_range, is_reverse_scan);
+
+  //20 dup exist
+  seeds.reset();
+  for (int64_t i = 0; i < 20; ++i) {
+    ret = seeds.push_back(0);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, count_per_range, is_reverse_scan);
+
+  // 20 not exist
+  seeds.reset();
+  for (int64_t i = 0; i < 20; ++i) {
+    ret = seeds.push_back(row_cnt_ + i);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, count_per_range, is_reverse_scan);
+
+  // partial not exist
+  seeds.reset();
+  for (int64_t i = 0; i < 20; ++i) {
+    ret = seeds.push_back(i % 2 ? i : i + row_cnt_);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  test_one_case(seeds, count_per_range, is_reverse_scan);
+  destroy_query_param();
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_multi_scan)
+{
+  bool is_reverse_scan = false;
+  for (int64_t i = 2; i < 20; i += 10) {
+    test_multi_scan_multi_scan_range(is_reverse_scan, i);
+  }
+  is_reverse_scan = true;
+  for (int64_t i = 2; i < 20; i += 10) {
+    test_multi_scan_multi_scan_range(is_reverse_scan, i);
+  }
+}
+
+void TestDDLMergeRowMultiScanner::test_multi_scan_multi_get_with_scan(
+    const bool is_reverse_scan,
+    const int64_t count_per_range)
+{
+  int ret = OB_SUCCESS;
+  ObDatumRange range;
+  ObArray<ObDatumRange> ranges;
+  ObDatumRow row;
+  ObDatumRowkey rowkey;
+  const ObDatumRow *prow = NULL;
+  const ObDatumRow *kv_prow = NULL;
+  int64_t row_cnt = 0;
+  ObSSTableRowMultiScanner<> scanner;
+  ObSSTableRowMultiScanner<> merge_ddl_scanner;
+
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  // multi scan interact with multi get
+  ObDatumRange mget_ranges[TEST_MULTI_GET_CNT];
+  ObDatumRow start_row;
+  ObDatumRow end_row;
+  ObDatumRow check_row;
+  ASSERT_EQ(OB_SUCCESS, start_row.init(allocator_, TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, end_row.init(allocator_, TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, check_row.init(allocator_, TEST_COLUMN_CNT));
+  ranges.reuse();
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ObDatumRowkey tmp_rowkey;
+    mget_ranges[i].border_flag_.set_inclusive_start();
+    mget_ranges[i].border_flag_.set_inclusive_end();
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, start_row));
+    tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i % 2 ? count_per_range - 1 : 0), end_row));
+    tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
+  }
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ret = ranges.push_back(mget_ranges[i]);
+    ASSERT_EQ(OB_SUCCESS, ret);
+  }
+  STORAGE_LOG(INFO, "multi scan begin");
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+          iter_param_,
+          context_,
+          &sstable_,
+          &ranges));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+          iter_param_,
+          context_,
+          &partial_sstable_,
+          &ranges));
+  row_cnt = 0;
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    const int64_t p = i;
+    if (p % 2) {
+      for (int64_t j = 0; j < count_per_range; ++j) {
+        const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
+        ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
+        ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_get_next_row(kv_prow));
+        ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(k, check_row));
+        ++row_cnt;
+        ASSERT_TRUE(*prow == check_row);
+        ASSERT_TRUE(*kv_prow == check_row);
+      }
+    } else {
+      ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
+      ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_get_next_row(kv_prow));
+      ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(p, check_row));
+      ++row_cnt;
+      ASSERT_TRUE(*prow == check_row);
+      ASSERT_TRUE(*kv_prow == check_row);
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  scanner.reuse();
+  merge_ddl_scanner.reuse();
+
+  // first half multi scan, second half multi get
+  ranges.reuse();
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ObDatumRowkey tmp_rowkey;
+    mget_ranges[i].border_flag_.set_inclusive_start();
+    mget_ranges[i].border_flag_.set_inclusive_end();
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, start_row));
+    tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i < TEST_MULTI_GET_CNT / 2 ? count_per_range - 1 : 0), end_row));
+    tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, ranges.push_back(mget_ranges[i]));
+  }
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+          iter_param_,
+          context_,
+          &sstable_,
+          &ranges));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+          iter_param_,
+          context_,
+          &partial_sstable_,
+          &ranges));
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    const int64_t p = i;
+    if (p < TEST_MULTI_GET_CNT / 2) {
+      for (int64_t j = 0; j < count_per_range; ++j) {
+        const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
+        ret = scanner.inner_get_next_row(prow);
+        ASSERT_EQ(OB_SUCCESS, ret);
+        ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+        ASSERT_EQ(OB_SUCCESS, ret);
+        ret = row_generate_.get_next_row(k, check_row);
+        ASSERT_EQ(OB_SUCCESS, ret);
+        ASSERT_TRUE(*prow == check_row);
+        ASSERT_TRUE(*kv_prow == check_row);
+      }
+    } else {
+      ret = scanner.inner_get_next_row(prow);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ret = row_generate_.get_next_row(p, check_row);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ASSERT_TRUE(*prow == check_row);
+      ASSERT_TRUE(*kv_prow == check_row);
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  scanner.reuse();
+  merge_ddl_scanner.reuse();
+
+  // first one multi get, others multi scan
+  ranges.reuse();
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ObDatumRowkey tmp_rowkey;
+    mget_ranges[i].border_flag_.set_inclusive_start();
+    mget_ranges[i].border_flag_.set_inclusive_end();
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, start_row));
+    tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i != 0 ? count_per_range - 1 : 0), end_row));
+    tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, ranges.push_back(mget_ranges[i]));
+  }
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+          iter_param_,
+          context_,
+          &sstable_,
+          &ranges));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+          iter_param_,
+          context_,
+          &partial_sstable_,
+          &ranges));
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    const int64_t p = i;
+    if (p != 0) {
+      for (int64_t j = 0; j < count_per_range; ++j) {
+        const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
+        ret = scanner.inner_get_next_row(prow);
+        ASSERT_EQ(OB_SUCCESS, ret);
+        ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+        ASSERT_EQ(OB_SUCCESS, ret);
+        ret = row_generate_.get_next_row(k, check_row);
+        ASSERT_EQ(OB_SUCCESS, ret);
+        ASSERT_TRUE(*prow == check_row);
+        ASSERT_TRUE(*kv_prow == check_row);
+      }
+    } else {
+      ret = scanner.inner_get_next_row(prow);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ret = row_generate_.get_next_row(p, check_row);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ASSERT_TRUE(*prow == check_row);
+      ASSERT_TRUE(*kv_prow == check_row);
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  scanner.reuse();
+  merge_ddl_scanner.reuse();
+
+  // multi scan not exist row
+  STORAGE_LOG(DEBUG, "multi_scan_not_exist_row");
+  ranges.reuse();
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    ObDatumRowkey tmp_rowkey;
+    mget_ranges[i].border_flag_.set_inclusive_start();
+    mget_ranges[i].border_flag_.set_inclusive_end();
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i % 2 ? row_cnt_ : 0), start_row));
+    tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i % 2 ? row_cnt_ + count_per_range - 1 : 0), end_row));
+    tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, ranges.push_back(mget_ranges[i]));
+  }
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+          iter_param_,
+          context_,
+          &sstable_,
+          &ranges));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+          iter_param_,
+          context_,
+          &partial_sstable_,
+          &ranges));
+  for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
+    const int64_t p = i;
+    if (p % 2) {
+      continue;
+    } else {
+      ret = scanner.inner_get_next_row(prow);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ret = row_generate_.get_next_row(p, check_row);
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ASSERT_TRUE(*prow == check_row);
+      ASSERT_TRUE(*kv_prow == check_row);
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  scanner.reset();
+  destroy_query_param();
+}
+
+TEST_F(TestDDLMergeRowMultiScanner, test_multi_get_with_scan)
+{
+  bool is_reverse_scan = false;
+  for (int64_t i = 2; i < 10; i += 10) {
+    test_multi_scan_multi_get_with_scan(is_reverse_scan, i);
+  }
+  is_reverse_scan = true;
+  for (int64_t i = 2; i < 10; i += 10) {
+    test_multi_scan_multi_get_with_scan(is_reverse_scan, i);
+  }
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_ddl_merge_row_multi_scanner.log*");
+  OB_LOGGER.set_file_name("test_ddl_merge_row_multi_scanner.log", true, true);
+  oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

mittest/mtlenv/storage/blocksstable/test_ddl_merge_row_scanner.cpp

@@ -0,0 +1,537 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#define private public
+#define protected public
+
+#include "lib/random/ob_random.h"
+#include "storage/access/ob_index_tree_prefetcher.h"
+#include "storage/access/ob_sstable_row_getter.h"
+#include "storage/access/ob_sstable_row_scanner.h"
+#include "ob_index_block_data_prepare.h"
+
+namespace oceanbase
+{
+using namespace storage;
+using namespace common;
+namespace blocksstable
+{
+class TestDDLMergeRowScanner : public TestIndexBlockDataPrepare
+{
+public:
+  TestDDLMergeRowScanner();
+  virtual ~TestDDLMergeRowScanner();
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+
+  virtual void SetUp();
+  virtual void TearDown();
+
+  void generate_range(const int64_t start, const int64_t end, ObDatumRange &range);
+  void test_one_rowkey(const int64_t seed); //get
+  void test_single_case(
+      const ObDatumRange &range,
+      const int64_t start,
+      const int64_t end,
+      const bool is_reverse_scan);
+  void test_full_case(
+      const ObDatumRange &range,
+      const int64_t start,
+      const int64_t end,
+      const bool is_reverse_scan,
+      const int64_t hit_mode);
+  void test_border(const bool is_reverse_scan);
+  void test_basic(const bool is_reverse_scan);
+
+protected:
+  enum CacheHitMode
+  {
+    HIT_ALL = 0,
+    HIT_NONE,
+    HIT_PART,
+    HIT_MAX,
+  };
+private:
+  ObArenaAllocator allocator_;
+  ObDatumRow start_row_;
+  ObDatumRow end_row_;
+};
+
+TestDDLMergeRowScanner::TestDDLMergeRowScanner()
+  : TestIndexBlockDataPrepare("Test DDL multi row scanner")
+{
+  is_ddl_merge_data_ = true;
+  max_row_cnt_ = 150000;
+  max_partial_row_cnt_ = 78881;
+  partial_kv_start_idx_ = 3;
+}
+
+TestDDLMergeRowScanner::~TestDDLMergeRowScanner()
+{
+
+}
+
+void TestDDLMergeRowScanner::SetUpTestCase()
+{
+  TestIndexBlockDataPrepare::SetUpTestCase();
+}
+
+void TestDDLMergeRowScanner::TearDownTestCase()
+{
+  TestIndexBlockDataPrepare::TearDownTestCase();
+}
+
+void TestDDLMergeRowScanner::SetUp()
+{
+  TestIndexBlockDataPrepare::SetUp();
+  ASSERT_EQ(OB_SUCCESS, start_row_.init(allocator_, TEST_COLUMN_CNT));
+  ASSERT_EQ(OB_SUCCESS, end_row_.init(allocator_, TEST_COLUMN_CNT));
+  ObLSID ls_id(ls_id_);
+  ObTabletID tablet_id(tablet_id_);
+  ObLSHandle ls_handle;
+  ObLSService *ls_svr = MTL(ObLSService*);
+  ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
+
+  ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
+}
+
+void TestDDLMergeRowScanner::TearDown()
+{
+  tablet_handle_.reset();
+  TestIndexBlockDataPrepare::TearDown();
+}
+
+void TestDDLMergeRowScanner::generate_range(
+    const int64_t start,
+    const int64_t end,
+    ObDatumRange &range)
+{
+  ObDatumRowkey tmp_rowkey;
+  ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start, start_row_));
+  tmp_rowkey.assign(start_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+  ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.start_key_, allocator_));
+  ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(end, end_row_));
+  tmp_rowkey.assign(end_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+  ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.end_key_, allocator_));
+  range.border_flag_.set_inclusive_start();
+  range.border_flag_.set_inclusive_end();
+}
+
+void TestDDLMergeRowScanner::test_one_rowkey(const int64_t seed)
+{
+  ObSSTableRowGetter getter;
+  ObSSTableRowGetter merge_ddl_getter;
+  ObDatumRow query_row;
+  ASSERT_EQ(OB_SUCCESS, query_row.init(allocator_, TEST_COLUMN_CNT));
+  row_generate_.get_next_row(seed, query_row);
+  ObDatumRowkey query_rowkey;
+  query_rowkey.assign(query_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+  STORAGE_LOG(INFO, "Query rowkey", K(query_row));
+  ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &query_rowkey));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_getter.inner_open(iter_param_, context_, &partial_sstable_, &query_rowkey));
+
+  const ObDatumRow *prow = nullptr;
+  const ObDatumRow *kv_prow = nullptr;
+  ASSERT_EQ(OB_SUCCESS, getter.inner_get_next_row(prow));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_getter.inner_get_next_row(kv_prow));
+  STORAGE_LOG(INFO, "debug datum row1", KPC(prow), KPC(kv_prow));
+  if (seed >= row_cnt_) {
+    ASSERT_TRUE(prow->row_flag_.is_not_exist());
+    ASSERT_TRUE(kv_prow->row_flag_.is_not_exist());
+  } else {
+    ASSERT_TRUE(*prow == query_row);
+    if (!(*kv_prow == query_row)) {
+      STORAGE_LOG(INFO, "CHECK UNEUQAL", K(query_row), K(*kv_prow));
+    }
+    ASSERT_TRUE(*kv_prow == query_row);
+  }
+  ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_getter.inner_get_next_row(kv_prow));
+  getter.reuse();
+  merge_ddl_getter.reuse();
+}
+
+void TestDDLMergeRowScanner::test_single_case(
+    const ObDatumRange &range,
+    const int64_t start,
+    const int64_t end,
+    const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObDatumRow row;
+  ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT));
+  const ObDatumRow *prow = nullptr;
+  const ObDatumRow *kv_prow = nullptr;
+  ObSSTableRowScanner<> scanner;
+  ObSSTableRowScanner<> merge_ddl_scanner;
+
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+          iter_param_,
+          context_,
+          &sstable_,
+          &range));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+          iter_param_,
+          context_,
+          &partial_sstable_,
+          &range));
+  for (int64_t i = start; i <= end; ++i) {
+    int64_t index = 0;
+    if (is_reverse_scan) {
+      ret = row_generate_.get_next_row(end - i + start, row);
+      index = end - i + start;
+    } else {
+      ret = row_generate_.get_next_row(i, row);
+      index = i;
+    }
+
+    ASSERT_EQ(OB_SUCCESS, ret);
+    if (index < row_cnt_) {
+      ret = scanner.inner_get_next_row(prow);
+      ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
+          << " end: " << end << " prow: " << prow;
+      ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
+          << " end: " << end << " prow: " << prow;
+
+      ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+      ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
+          << " end: " << end << " kv_prow: " << kv_prow;
+      ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
+          << " end: " << end << " kv_prow: " << kv_prow;
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  scanner.reuse();
+  merge_ddl_scanner.reuse();
+}
+
+void TestDDLMergeRowScanner::test_full_case(
+    const ObDatumRange &range,
+    const int64_t start,
+    const int64_t end,
+    const bool is_reverse_scan,
+    const int64_t hit_mode)
+{
+  int ret = OB_SUCCESS;
+  ObDatumRow row;
+  ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT));
+  const ObDatumRow *prow = nullptr;
+  const ObDatumRow *kv_prow = nullptr;
+  ObSSTableRowScanner<> scanner;
+  ObSSTableRowScanner<> merge_ddl_scanner;
+
+  if (HIT_PART == hit_mode) {
+    const int64_t part_start = start + (end - start) / 3;
+    const int64_t part_end = end - (end - start) / 3;
+    ObDatumRange part_range;
+    ObDatumRow start_row;
+    ObDatumRow end_row;
+    ObDatumRowkey tmp_rowkey;
+    ASSERT_EQ(OB_SUCCESS, start_row.init(allocator_, TEST_COLUMN_CNT));
+    ASSERT_EQ(OB_SUCCESS, end_row.init(allocator_, TEST_COLUMN_CNT));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(part_start, start_row));
+    tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(part_range.start_key_, allocator_));
+    ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(part_end, end_row));
+    tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
+    ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(part_range.end_key_, allocator_));
+    part_range.border_flag_.set_inclusive_start();
+    part_range.border_flag_.set_inclusive_end();
+    ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+            iter_param_,
+            context_,
+            &sstable_,
+            &part_range));
+    ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+            iter_param_,
+            context_,
+            &partial_sstable_,
+            &part_range));
+    for (int64_t i = part_start; i <= part_end; ++i) {
+      if (i < row_cnt_) {
+        ret = scanner.inner_get_next_row(prow);
+        ASSERT_EQ(OB_SUCCESS, ret) << "i: " << i << " part_start: " << part_start
+            << " part_end: " << part_end << " prow: " << prow;
+        ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+        ASSERT_EQ(OB_SUCCESS, ret) << "i: " << i << " part_start: " << part_start
+            << " part_end: " << part_end << " kv_prow: " << kv_prow;
+      }
+    }
+    ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+    ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+    scanner.reuse();
+    merge_ddl_scanner.reuse();
+  }
+
+  ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+          iter_param_,
+          context_,
+          &sstable_,
+          &range));
+  ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+          iter_param_,
+          context_,
+          &partial_sstable_,
+          &range));
+  for (int64_t i = start; i <= end; ++i) {
+    int64_t index = 0;
+    if (is_reverse_scan) {
+      ret = row_generate_.get_next_row(end - i + start, row);
+      index = end - i + start;
+    } else {
+      ret = row_generate_.get_next_row(i, row);
+      index = i;
+    }
+    ASSERT_EQ(OB_SUCCESS, ret);
+    if (index < row_cnt_) {
+      ret = scanner.inner_get_next_row(prow);
+      ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
+          << " end: " << end << " prow: " << prow;
+      ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
+          << " end: " << end << " prow: " << prow;
+      ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+      ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
+          << " end: " << end << " kv_prow: " << kv_prow;
+      ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
+          << " end: " << end << " kv_prow: " << kv_prow;
+    }
+  }
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+  scanner.reuse();
+  merge_ddl_scanner.reuse();
+
+  if (HIT_ALL == hit_mode) {
+    int64_t index = 0;
+    ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
+            iter_param_,
+            context_,
+            &sstable_,
+            &range));
+    ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
+            iter_param_,
+            context_,
+            &partial_sstable_,
+            &range));
+    for (int64_t i = start; i <= end; ++i) {
+      if (is_reverse_scan) {
+        ret = row_generate_.get_next_row(end - i + start, row);
+        index = end - i + start;
+      } else {
+        ret = row_generate_.get_next_row(i, row);
+        index = i;
+      }
+      ASSERT_EQ(OB_SUCCESS, ret);
+      if (index < row_cnt_) {
+        ret = scanner.inner_get_next_row(prow);
+        ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
+            << " end: " << end << " prow: " << prow;
+        ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
+            << " end: " << end << " prow: " << prow;
+        ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
+        ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
+            << " end: " << end << " kv_prow: " << kv_prow;
+        ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
+            << " end: " << end << " kv_prow: " << kv_prow;
+      }
+    }
+    ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+    ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
+    ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+    ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
+    scanner.reuse();
+    merge_ddl_scanner.reuse();
+  }
+}
+
+void TestDDLMergeRowScanner::test_border(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObDatumRange range;
+
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+
+  // full table scan
+  range.set_whole_range();
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, 0, row_cnt_ - 1, is_reverse_scan, i);
+  }
+
+  // the first row of sstable
+  generate_range(0, 0, range);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, 0, 0, is_reverse_scan, i);
+  }
+
+  // the first 100 row of sstable
+  generate_range(0, 100, range);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, 0, 100, is_reverse_scan, i);
+  }
+
+  // the last row of sstable
+  generate_range(row_cnt_ - 1, row_cnt_ - 1, range);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, row_cnt_ - 1, row_cnt_ - 1, is_reverse_scan, i);
+  }
+
+  // the last 100 row of sstable
+  generate_range(row_cnt_ - 100, row_cnt_ - 1, range);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, row_cnt_ - 100, row_cnt_ - 1, is_reverse_scan, i);
+  }
+
+  // not exist
+  generate_range(row_cnt_, row_cnt_, range);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, row_cnt_, row_cnt_, is_reverse_scan, i);
+  }
+
+  destroy_query_param();
+}
+
+void TestDDLMergeRowScanner::test_basic(const bool is_reverse_scan)
+{
+  int ret = OB_SUCCESS;
+  ObDatumRange range;
+
+  prepare_query_param(is_reverse_scan);
+  // full table scan
+  range.set_whole_range();
+  test_single_case(range, 0, row_cnt_ - 1, is_reverse_scan);
+
+  // the first row of sstable
+  generate_range(0, 0, range);
+  test_single_case(range, 0, 0, is_reverse_scan);
+
+  // the first 100 row of sstable
+  generate_range(0, 100, range);
+  test_single_case(range, 0, 100, is_reverse_scan);
+
+  // the last row of sstable
+  generate_range(row_cnt_ - 1, row_cnt_ - 1, range);
+  test_single_case(range, row_cnt_ - 1, row_cnt_ - 1, is_reverse_scan);
+
+  // the last 100 row of sstable
+  generate_range(row_cnt_ - 100, row_cnt_ - 1, range);
+  test_single_case(range, row_cnt_ - 100, row_cnt_ - 1, is_reverse_scan);
+
+  // not exist
+  generate_range(row_cnt_, row_cnt_, range);
+  test_single_case(range, row_cnt_, row_cnt_, is_reverse_scan);
+  destroy_query_param();
+}
+
+TEST_F(TestDDLMergeRowScanner, test_get)
+{
+  // forward get
+  prepare_query_param(false);
+  // left border rowkey
+  test_one_rowkey(0);
+  // end_key of left border
+  test_one_rowkey(15933);
+  // first_key of right border
+  test_one_rowkey(132361);
+  // end_key of right border
+  test_one_rowkey(row_cnt_ - 1);
+  // mid border rowkey
+  test_one_rowkey(row_cnt_ / 2);
+  // non-exist rowkey
+  test_one_rowkey(row_cnt_);
+  destroy_query_param();
+
+  // reverse get
+  prepare_query_param(true);
+  // left border rowkey
+  test_one_rowkey(0);
+  // end_key of left border
+  test_one_rowkey(15933);
+  // first_key of right border
+  test_one_rowkey(132361);
+  // end_key of right border
+  test_one_rowkey(row_cnt_ - 1);
+  // mid border rowkey
+  test_one_rowkey(row_cnt_ / 2);
+  // non-exist rowkey
+  test_one_rowkey(row_cnt_);
+  destroy_query_param();
+}
+
+TEST_F(TestDDLMergeRowScanner, test_basic_scan)
+{
+  bool is_reverse_scan = false;
+  test_basic(is_reverse_scan);
+  is_reverse_scan = true;
+  test_basic(is_reverse_scan);
+  STORAGE_LOG(INFO, "memory usage", K(lib::get_memory_hold()), K(lib::get_memory_limit()));
+  ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
+  ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
+  ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
+}
+
+TEST_F(TestDDLMergeRowScanner, test_border_scan)
+{
+  bool is_reverse_scan = false;
+  test_border(is_reverse_scan);
+  is_reverse_scan = true;
+  test_border(is_reverse_scan);
+  STORAGE_LOG(INFO, "memory usage", K(lib::get_memory_hold()), K(lib::get_memory_limit()));
+  ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
+  ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
+  ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
+}
+
+TEST_F(TestDDLMergeRowScanner, test_random_scan)
+{
+  ObDatumRange range;
+  int64_t start = ObRandom::rand(0, row_cnt_ - 1);
+  int64_t end = ObRandom::rand(0, row_cnt_ - 1);
+  if (start > end) {
+    int64_t temp = start;
+    start = end;
+    end = temp;
+  }
+  generate_range(start, end, range);
+
+  bool is_reverse_scan = false;
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, start, end, is_reverse_scan, i);
+  }
+  destroy_query_param();
+
+  is_reverse_scan = true;
+  // prepare query param
+  prepare_query_param(is_reverse_scan);
+  for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
+    test_full_case(range, start, end, is_reverse_scan, i);
+  }
+  destroy_query_param();
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_ddl_merge_row_scanner.log*");
+  OB_LOGGER.set_file_name("test_ddl_merge_row_scanner.log", true, true);
+  oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

mittest/mtlenv/storage/blocksstable/test_direct_load.cpp

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#define private public
+#define protected public
+
+#include "ob_index_block_data_prepare.h"
+#include "storage/ddl/ob_direct_insert_sstable_ctx_new.h"
+
+namespace oceanbase
+{
+using namespace lib;
+using namespace common;
+using namespace share;
+using namespace storage;
+using namespace blocksstable;
+
+class TestDirectLoad : public TestIndexBlockDataPrepare
+{
+public:
+  TestDirectLoad() : TestIndexBlockDataPrepare("Test direct load") {}
+  virtual ~TestDirectLoad() {}
+  static void SetUpTestCase() { TestIndexBlockDataPrepare::SetUpTestCase(); }
+  static void TearDownTestCase() { TestIndexBlockDataPrepare::TearDownTestCase(); }
+
+  virtual void SetUp();
+  virtual void TearDown();
+};
+
+void TestDirectLoad::SetUp()
+{
+  TestIndexBlockDataPrepare::SetUp();
+  ObLSID ls_id(ls_id_);
+  ObTabletID tablet_id(tablet_id_);
+  ObLSHandle ls_handle;
+  ObLSService *ls_svr = MTL(ObLSService*);
+  ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
+  ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
+
+  prepare_query_param(true);
+}
+
+void TestDirectLoad::TearDown()
+{
+  destroy_query_param();
+  tablet_handle_.reset();
+  TestIndexBlockDataPrepare::TearDown();
+}
+
+TEST_F(TestDirectLoad, init_ddl_table_store)
+{
+  ObTabletFullDirectLoadMgr tablet_dl_mgr;
+  ObTabletDirectLoadInsertParam build_param;
+  build_param.common_param_.ls_id_ = ls_id_;
+  build_param.common_param_.tablet_id_ = tablet_id_;
+  build_param.common_param_.direct_load_type_ = ObDirectLoadType::DIRECT_LOAD_DDL;
+  build_param.common_param_.read_snapshot_ = SNAPSHOT_VERSION;
+  build_param.runtime_only_param_.task_cnt_ = 1;
+  build_param.runtime_only_param_.task_id_ = 1;
+  build_param.runtime_only_param_.table_id_ = TEST_TABLE_ID;
+  build_param.runtime_only_param_.schema_version_ = 1;
+  SCN ddl_start_scn;
+  ASSERT_EQ(OB_SUCCESS, ddl_start_scn.convert_from_ts(ObTimeUtility::current_time()));
+  ASSERT_EQ(OB_SUCCESS, tablet_dl_mgr.update(nullptr, build_param));
+  tablet_dl_mgr.start_scn_ = ddl_start_scn;
+  tablet_dl_mgr.data_format_version_ = DATA_VERSION_4_0_0_0;
+  ASSERT_EQ(OB_SUCCESS, tablet_dl_mgr.init_ddl_table_store(ddl_start_scn, SNAPSHOT_VERSION, ddl_start_scn));
+
+}
+
+
+} // namespace oceanbase
+
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_direct_load.log*");
+  OB_LOGGER.set_file_name("test_direct_load.log", true, true);
+  oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp

@@ -204,7 +204,6 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan)
       ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID,
       *raw_block,
       root_blk_header->rowkey_array_[root_row_id]));
-  ASSERT_EQ(idx_scanner.current_, raw_idx_scanner.current_);
 
   idx_scanner.reuse();
   raw_idx_scanner.reuse();
@@ -224,7 +223,6 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan)
 
   int tmp_ret = OB_SUCCESS;
   while (OB_SUCCESS == tmp_ret) {
-    ASSERT_EQ(idx_scanner.current_, raw_idx_scanner.current_);
     tmp_ret = idx_scanner.get_next(read_idx_info);
     if (OB_SUCCESS == tmp_ret) {
       ASSERT_EQ(tmp_ret, raw_idx_scanner.get_next(raw_read_idx_info));

mittest/mtlenv/storage/blocksstable/test_sstable_row_getter.cpp

@@ -113,7 +113,13 @@ TEST_F(TestSSTableRowGetter, get)
   //left border rowkey
   test_one_rowkey(0);
 
-  // right border rowkey
+  //end_key of left border
+  test_one_rowkey(15933);
+
+  //first_key of right border
+  test_one_rowkey(64760);
+
+  // end_key of right border
   test_one_rowkey(row_cnt_ - 1);
 
   // mid border rowkey

mittest/mtlenv/storage/test_co_merge.cpp

@@ -299,6 +299,7 @@ void TestCOMerge::prepare_scan_param(
   iter_param.vectorized_enabled_ = false;
   ASSERT_EQ(OB_SUCCESS,
             store_ctx.init_for_read(ls_id,
+                                    iter_param.tablet_id_,
                                     INT64_MAX, // query_expire_ts
                                     -1, // lock_timeout_us
                                     share::SCN::max_scn()));
@@ -436,6 +437,7 @@ void TestCOMerge::prepare_query_param(const ObVersionRange &version_range)
   iter_param_.vectorized_enabled_ = false;
   ASSERT_EQ(OB_SUCCESS,
             store_ctx_.init_for_read(ls_id,
+                                     iter_param_.tablet_id_,
                                      INT64_MAX, // query_expire_ts
                                      -1, // lock_timeout_us
                                      share::SCN::max_scn()));

mittest/mtlenv/storage/test_index_sstable_estimator.cpp

@@ -31,7 +31,7 @@ namespace unittest
 class TestIndexSSTableEstimator : public TestIndexBlockDataPrepare
 {
 public:
-  TestIndexSSTableEstimator() : TestIndexBlockDataPrepare("Test index sstable estimator") {}
+  TestIndexSSTableEstimator();
   virtual ~TestIndexSSTableEstimator() {}
   virtual void SetUp();
   virtual void TearDown();
@@ -46,6 +46,14 @@ private:
   ObDatumRow end_row_;
 };
 
+TestIndexSSTableEstimator::TestIndexSSTableEstimator()
+  : TestIndexBlockDataPrepare("Test index sstable estimator")
+{
+  is_ddl_merge_data_ = true;
+  max_row_cnt_ = 150000;
+  max_partial_row_cnt_ = 78881;
+  partial_kv_start_idx_ = 3;
+}
 void TestIndexSSTableEstimator::SetUpTestCase()
 {
   TestIndexBlockDataPrepare::SetUpTestCase();
@@ -113,12 +121,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_whole_range)
   range.set_whole_range();
 
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
+  ASSERT_EQ(part_est, ddl_merge_part_est);
 }
 
 TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_range)
@@ -126,12 +143,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_range)
   ObDatumRange range;
   generate_range(100, -1, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
+  ASSERT_EQ(part_est, ddl_merge_part_est);
 }
 
 TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_left_range)
@@ -139,12 +165,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_left_range)
   ObDatumRange range;
   generate_range(-1, 100, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
+  ASSERT_EQ(part_est, ddl_merge_part_est);
 }
 
 TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_right_range)
@@ -152,12 +187,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_right_range)
   ObDatumRange range;
   generate_range(row_cnt_ - 100, -1, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
+  ASSERT_EQ(part_est, ddl_merge_part_est);
 }
 
 TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_middle_range)
@@ -165,12 +209,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_middle_range)
   ObDatumRange range;
   generate_range(100, row_cnt_ - 100, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
+  ASSERT_EQ(part_est, ddl_merge_part_est);
 }
 
 TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_noexist_range)
@@ -178,12 +231,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_noexist_range)
   ObDatumRange range;
   generate_range(row_cnt_, row_cnt_, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
+  ASSERT_EQ(part_est, ddl_merge_part_est);
 }
 
 }  // end namespace unittest

mittest/mtlenv/storage/test_index_sstable_multi_estimator.cpp

@@ -31,7 +31,7 @@ namespace unittest
 class TestMultiVersionIndexSSTableEstimator : public TestIndexBlockDataPrepare
 {
 public:
-  TestMultiVersionIndexSSTableEstimator() : TestIndexBlockDataPrepare("Test multi version index sstable estimator", compaction::MINI_MERGE) {}
+  TestMultiVersionIndexSSTableEstimator();
   virtual ~TestMultiVersionIndexSSTableEstimator() {}
   virtual void SetUp();
   virtual void TearDown();
@@ -48,6 +48,15 @@ private:
   ObDatumRow end_row_;
 };
 
+TestMultiVersionIndexSSTableEstimator::TestMultiVersionIndexSSTableEstimator()
+  : TestIndexBlockDataPrepare("Test multi version index sstable estimator", MINI_MERGE)
+{
+  is_ddl_merge_data_ = true;
+  max_row_cnt_ = 150000;
+  max_partial_row_cnt_ = 137312;
+  partial_kv_start_idx_ = 29;
+}
+
 void TestMultiVersionIndexSSTableEstimator::SetUpTestCase()
 {
   TestIndexBlockDataPrepare::SetUpTestCase();
@@ -114,12 +123,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_minor_sstable_whole_range
   ObDatumRange range;
   range.set_whole_range();
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
 }
 
 TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_minor_sstable_range)
@@ -127,12 +144,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_minor_sstable_range)
   ObDatumRange range;
   generate_range(100, -1, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
 }
 
 TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_left_range)
@@ -140,12 +165,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_left_range)
   ObDatumRange range;
   generate_range(-1, 100, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
 }
 
 TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_right_range)
@@ -153,12 +186,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_right_range
   ObDatumRange range;
   generate_range(row_cnt_ - 100, -1, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
 }
 
 TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_middle_range)
@@ -166,12 +207,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_middle_rang
   ObDatumRange range;
   generate_range(100, row_cnt_ - 100, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
 }
 
 TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_noexist_range)
@@ -179,12 +228,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_noexist_ran
   ObDatumRange range;
   generate_range(row_cnt_, row_cnt_, range);
   ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
+  ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
 
   ObPartitionEst part_est;
   ObIndexBlockScanEstimator estimator(esti_ctx);
+  ObPartitionEst ddl_kv_part_est;
+  ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
+  ObPartitionEst ddl_merge_part_est;
+  ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
 
   ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
-  STORAGE_LOG(INFO, "part_est", K(part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
+  ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
+  STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
 }
 
 

mittest/mtlenv/storage/test_lob_meta_iterator.cpp

@@ -0,0 +1,204 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include <codecvt>
+#include <stdio.h>
+#define private public
+#define protected public
+
+#include "common/object/ob_obj_type.h"
+#include "common/object/ob_object.h"
+#include "share/schema/ob_table_param.h"
+#include "storage/blocksstable/ob_datum_row.h"
+#include "storage/lob/ob_lob_util.h"
+#include "storage/lob/ob_lob_meta.h"
+#include "lib/number/ob_number_v2.h"
+#include "share/schema/ob_column_schema.h"
+#include "share/ob_ls_id.h"
+#include "mtlenv/mock_tenant_module_env.h"
+
+namespace oceanbase
+{
+namespace storage
+{
+
+class TestLobMetaIterator : public ::testing::Test
+{
+public:
+  TestLobMetaIterator() = default;
+  virtual ~TestLobMetaIterator() = default;
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+
+  int fill_lob_sstable_slice_mock(
+                          const ObLobId &lob_id,
+                          const transaction::ObTransID &trans_id,
+                          const int64_t trans_version,
+                          const int64_t sql_no,
+                          const bool has_lob_header,
+                          const int64_t read_snapshot,
+                          const ObCollationType collation_type,
+                          blocksstable::ObStorageDatum &datum);
+  int build_lob_data(ObObj &obj, std::string &st, common::ObArenaAllocator &allocator);
+  int build_lob_data_not_common(ObObj &obj, std::string &str);
+};
+
+void TestLobMetaIterator::SetUpTestCase()
+{
+  EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
+}
+
+void TestLobMetaIterator::TearDownTestCase()
+{
+  MockTenantModuleEnv::get_instance().destroy();
+}
+
+int TestLobMetaIterator::fill_lob_sstable_slice_mock(
+                          const ObLobId &lob_id,
+                          const transaction::ObTransID &trans_id,
+                          const int64_t trans_version,
+                          const int64_t sql_no,
+                          const bool has_lob_header,
+                          const int64_t read_snapshot,
+                          const ObCollationType collation_type,
+                          blocksstable::ObStorageDatum &datum)
+{
+  int ret = OB_SUCCESS;
+  int64_t ils_id = 1001;
+  int64_t itablet_id = 1001;
+  common::ObArenaAllocator allocator;
+  share::ObLSID ls_id(ils_id);
+  common::ObTabletID tablet_id(itablet_id);
+  const blocksstable::ObDatumRow *new_row = nullptr;
+  ObLobMetaInfo lob_meta_info;
+  const int64_t timeout_ts = ObTimeUtility::current_time();
+  ObString data = datum.get_string();
+  ObLobMetaWriteIter iter(data, &allocator, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE);
+  ObLobMetaRowIterator row_iterator;
+  if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column(
+    allocator, nullptr, ls_id, tablet_id, lob_id, collation_type, datum, timeout_ts, has_lob_header, iter))) {
+    STORAGE_LOG(WARN, "fail to insert lob column", K(ret), K(ls_id), K(tablet_id), K(lob_id));
+  } else if (OB_FAIL(row_iterator.init(&iter, trans_id, trans_version, sql_no))) {
+    STORAGE_LOG(WARN, "fail to init lob meta row iterator", K(ret), K(trans_id), K(trans_version));
+  } else if (OB_FAIL(row_iterator.get_next_row(new_row))) {
+    STORAGE_LOG(WARN, "get_next_row failed", K(ret));
+  } else if (OB_ISNULL(new_row)) {
+    ret = OB_ERR_UNEXPECTED;
+    STORAGE_LOG(WARN, "new_row is nullptr", K(ret));
+  } else if (OB_FAIL(ObLobMetaUtil::transform_from_row_to_info(new_row, lob_meta_info, true))) {
+    STORAGE_LOG(WARN, "transform failed", K(ret));
+  } else if (lob_meta_info.lob_id_.lob_id_ != lob_id.lob_id_) {
+    //STORAGE_LOG("lob_meta_info", K(lob_meta_info));
+    STORAGE_LOG(WARN, "error info", K(ret), K(lob_meta_info.lob_id_.lob_id_), K(lob_id.lob_id_));
+  }
+  return ret;
+}
+int TestLobMetaIterator::build_lob_data(ObObj &obj, std::string &st, common::ObArenaAllocator &allocator)
+{
+  int ret = OB_SUCCESS;
+  ObLobCommon *value = NULL;
+  void *buf = NULL;
+  if (OB_ISNULL(buf = allocator.alloc(sizeof(ObLobCommon) + 1000000))) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    STORAGE_LOG(WARN, "fail to allocate memory for ObLobData", K(ret));
+  } else {
+    // ObLobIndex index;
+    value = new (buf) ObLobCommon();
+    // value->version_ = 1;
+    // value->reserve_ = 0;
+    // value->is_init_ = 1;
+    int64_t byte_size = 1000000;
+    value->in_row_ = 1;
+    MEMCPY(value->buffer_, st.c_str(), st.length());
+
+    obj.meta_.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI);
+    obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT);
+    obj.set_type(ObMediumTextType);
+    obj.set_lob_value(ObMediumTextType, value, value->get_handle_size(byte_size));
+    obj.set_has_lob_header();
+  }
+  return ret;
+}
+
+int TestLobMetaIterator::build_lob_data_not_common(ObObj &obj, std::string &str)
+{
+  int ret = OB_SUCCESS;
+  obj.meta_.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI);
+  obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT);
+  obj.set_type(ObMediumTextType);
+  obj.set_lob_value(ObMediumTextType, str.c_str(), str.length());
+  return ret;
+}
+
+TEST_F(TestLobMetaIterator, test_not_lob_common)
+{
+  for (int x = 2; x <= 10; x++) {
+    std::string st = "";
+    for (int i = 0; i < 1000000; i++) {
+      st += static_cast<char>(i % 26 + 'a');
+    }
+    ObDatumRow row;
+    ObObj obj;
+    ASSERT_EQ(OB_SUCCESS, build_lob_data_not_common(obj, st));
+    ObStorageDatum datum;
+    ASSERT_EQ(OB_SUCCESS, datum.from_obj_enhance(obj));
+    transaction::ObTransID trans_id(10);
+    ObLobId lob_id;
+    lob_id.lob_id_ = x;
+    lob_id.tablet_id_ = x;
+    int64_t sql_no = 2;
+    int64_t read_snapshot = 3;
+    ObCollationType collation_type = CS_TYPE_UTF8MB4_GENERAL_CI;
+    ASSERT_EQ(OB_SUCCESS, fill_lob_sstable_slice_mock(lob_id, trans_id, 100, sql_no,
+                                                      false, read_snapshot, collation_type, datum));
+  }
+}
+
+TEST_F(TestLobMetaIterator, test_lob_common)
+{
+  for (int x = 2; x <= 10; x++) {
+    std::string st = "";
+    for (int i = 0; i < 1000000; i++) {
+      st += static_cast<char>(i % 26 + 'a');
+    }
+    ObDatumRow row;
+    ObObj obj;
+    common::ObArenaAllocator allocator;
+    ASSERT_EQ(OB_SUCCESS, build_lob_data(obj, st, allocator));
+    ObStorageDatum datum;
+    ASSERT_EQ(OB_SUCCESS, datum.from_obj_enhance(obj));
+    transaction::ObTransID trans_id(10);
+    ObLobId lob_id;
+    lob_id.lob_id_ = x;
+    lob_id.tablet_id_ = x;
+    int64_t sql_no = 2;
+    int64_t read_snapshot = 3;
+    //ObLobMetaRowIterator iter;
+    ObCollationType collation_type = CS_TYPE_UTF8MB4_GENERAL_CI;
+    ASSERT_EQ(OB_SUCCESS, fill_lob_sstable_slice_mock(lob_id, trans_id, 100, sql_no,
+                                                      true, read_snapshot, collation_type, datum));
+  }
+}
+}
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_lob_meta_iterator.log*");
+  OB_LOGGER.set_file_name("test_lob_meta_iterator.log");
+  oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

mittest/mtlenv/storage/test_ls_migration_param.cpp

@@ -239,15 +239,17 @@ TEST_F(TestLSMigrationParam, test_migrate_tablet_param)
   TestSchemaUtils::prepare_data_schema(table_schema);
   ObArenaAllocator schema_allocator;
   ObCreateTabletSchema create_tablet_schema;
+
   ret = create_tablet_schema.init(schema_allocator, table_schema, lib::Worker::CompatMode::MYSQL,
-        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
+        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
+        0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
   ASSERT_EQ(OB_SUCCESS, ret);
 
   ObTabletID empty_tablet_id;
   SCN scn;
   scn.convert_from_ts(ObTimeUtility::current_time());
   ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
-      scn, 2022, create_tablet_schema, false, ls_handle.get_ls()->get_freezer());
+      scn, 2022, create_tablet_schema, ls_handle.get_ls()->get_freezer());
   ASSERT_EQ(common::OB_SUCCESS, ret);
 
   ObMigrationTabletParam tablet_param;
@@ -303,15 +305,17 @@ TEST_F(TestLSMigrationParam, test_migration_param_compat)
   TestSchemaUtils::prepare_data_schema(table_schema);
   ObArenaAllocator schema_allocator;
   ObCreateTabletSchema create_tablet_schema;
+
   ret = create_tablet_schema.init(schema_allocator, table_schema, lib::Worker::CompatMode::MYSQL,
-        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
+        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
+        0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
   ASSERT_EQ(OB_SUCCESS, ret);
 
   ObTabletID empty_tablet_id;
   SCN scn;
   scn.convert_from_ts(ObTimeUtility::current_time());
   ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
-      scn, 2022, create_tablet_schema, false, ls_handle.get_ls()->get_freezer());
+      scn, 2022, create_tablet_schema, ls_handle.get_ls()->get_freezer());
   ASSERT_EQ(common::OB_SUCCESS, ret);
 
   ObMigrationTabletParam tablet_param;

mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp

@@ -156,15 +156,17 @@ void TestLSTabletInfoWR::fill_tablet_meta()
   TestSchemaUtils::prepare_data_schema(table_schema);
   ObArenaAllocator schema_allocator;
   ObCreateTabletSchema create_tablet_schema;
+
   ret = create_tablet_schema.init(schema_allocator, table_schema, lib::Worker::CompatMode::MYSQL,
-        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
+        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
+        0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
   ASSERT_EQ(OB_SUCCESS, ret);
 
   ObTabletID empty_tablet_id;
   SCN scn;
   scn.convert_from_ts(ObTimeUtility::current_time());
   ret = src_handle.get_obj()->init_for_first_time_creation(arena_allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
-      scn, 2022, create_tablet_schema, false, ls_handle.get_ls()->get_freezer());
+      scn, 2022, create_tablet_schema, ls_handle.get_ls()->get_freezer());
   ASSERT_EQ(common::OB_SUCCESS, ret);
 
   ObMigrationTabletParam tablet_param;

mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp

@@ -180,6 +180,7 @@ void TestMultiVersionMergeRecycle::prepare_query_param(const ObVersionRange &ver
   iter_param_.vectorized_enabled_ = false;
   ASSERT_EQ(OB_SUCCESS,
             store_ctx_.init_for_read(ls_id,
+                                     iter_param_.tablet_id_,
                                      INT64_MAX, // query_expire_ts
                                      -1, // lock_timeout_us
                                      INT64_MAX - 2));

mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp

@@ -198,6 +198,7 @@ void TestMultiVersionMerge::prepare_query_param(const ObVersionRange &version_ra
   iter_param_.vectorized_enabled_ = false;
   ASSERT_EQ(OB_SUCCESS,
             store_ctx_.init_for_read(ls_id,
+                                     iter_param_.tablet_id_,
                                      INT64_MAX, // query_expire_ts
                                      -1, // lock_timeout_us
                                      share::SCN::max_scn()));

mittest/mtlenv/storage/test_multi_version_sstable_single_get.cpp

@@ -95,6 +95,7 @@ void TestMultiVersionSSTableSingleGet::prepare_query_param(
   iter_param_.vectorized_enabled_ = false;
   ASSERT_EQ(OB_SUCCESS,
             store_ctx_.init_for_read(ls_id,
+                                     iter_param_.tablet_id_,
                                      INT64_MAX, // query_expire_ts
                                      -1, // lock_timeout_us
                                      share::SCN::max_scn()));

mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp

@@ -2880,8 +2880,8 @@ TEST_F(TestTabletCreateDeleteHelper, migrate_lob_tablets)
     const ObSArray<ObTabletID> index_tablet_array;
     const ObTabletID tablet_id(2);
     const ObTabletID lob_meta_tablet_id(101);
-    ret = helper.do_create_tablet(lob_meta_tablet_id, tablet_id, lob_meta_tablet_id, ObTabletID(),
-        index_tablet_array, arg, trans_flags, table_schema1, lib::Worker::CompatMode::MYSQL, tablet_handle);
+    ret = helper.create_tablet(lob_meta_tablet_id, tablet_id, lob_meta_tablet_id, ObTabletID(),
+        index_tablet_array, arg, trans_flags, table_schema1, true /*need_create_empty_major_sstable*/, lib::Worker::CompatMode::MYSQL);
     ASSERT_EQ(OB_SUCCESS, ret);
     // reset tx data to normal state
     tablet_handle.get_obj()->tablet_meta_.tx_data_.tx_id_ = 0;
@@ -2889,8 +2889,8 @@ TEST_F(TestTabletCreateDeleteHelper, migrate_lob_tablets)
     tablet_handle.get_obj()->tablet_meta_.tx_data_.tx_scn_ = share::SCN::minus(share::SCN::max_scn(), 98);
 
     const ObTabletID lob_piece_tablet_id(102);
-    ret = helper.do_create_tablet(lob_piece_tablet_id, tablet_id, ObTabletID(), lob_piece_tablet_id,
-        index_tablet_array, arg, trans_flags, table_schema2, lib::Worker::CompatMode::MYSQL, tablet_handle);
+    ret = helper.create_tablet(lob_piece_tablet_id, tablet_id, ObTabletID(), lob_piece_tablet_id,
+        index_tablet_array, arg, trans_flags, table_schema2, true /*need_create_empty_major_sstable*/, lib::Worker::CompatMode::MYSQL);
     ASSERT_EQ(OB_SUCCESS, ret);
     // reset tx data to normal state
     tablet_handle.get_obj()->tablet_meta_.tx_data_.tx_id_ = 0;

mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp

@@ -226,7 +226,8 @@ void TestTenantMetaMemMgr::prepare_data_schema(
   LOG_INFO("dump data table schema", LITERAL_K(TEST_ROWKEY_COLUMN_CNT), K(table_schema));
 
   ret = create_tablet_schema.init(allocator, table_schema, lib::Worker::CompatMode::MYSQL,
-        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
+        false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
+        0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
   ASSERT_EQ(OB_SUCCESS, ret);
 }
 
@@ -704,8 +705,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_tablet)
   create_scn.convert_from_ts(ObTimeUtility::fast_current_time());
 
   ObTabletID empty_tablet_id;
+  create_tablet_schema.set_need_create_empty_major(true);
   ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
-      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, true, &freezer);
+      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
   ASSERT_EQ(common::OB_SUCCESS, ret);
   ASSERT_EQ(1, tablet->get_ref());
   ObTabletPersister persister;
@@ -799,8 +801,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_inner_tablet)
 
   ObTabletID empty_tablet_id;
   bool make_empty_co_sstable = true;
+  create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
   ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
-      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, make_empty_co_sstable, &freezer);
+      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
   ASSERT_EQ(common::OB_SUCCESS, ret);
   ASSERT_EQ(1, tablet->get_ref());
 
@@ -905,9 +908,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_no_sstable_tablet)
 
   ObTabletID empty_tablet_id;
   bool make_empty_co_sstable = false;
+  create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
   ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
-      create_scn, create_scn.get_val_for_tx(), create_tablet_schema,
-      make_empty_co_sstable, &freezer);
+      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
   ASSERT_EQ(common::OB_SUCCESS, ret);
   ASSERT_EQ(1, tablet->get_ref());
 
@@ -999,8 +1002,9 @@ TEST_F(TestTenantMetaMemMgr, test_get_tablet_with_allocator)
 
   ObTabletID empty_tablet_id;
   bool make_empty_co_sstable = true;
+  create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
   ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
-      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, make_empty_co_sstable, &freezer);
+      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
   ASSERT_EQ(common::OB_SUCCESS, ret);
   ASSERT_EQ(1, tablet->get_ref());
 
@@ -1123,8 +1127,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_mem_tablet)
 
   ObTabletID empty_tablet_id;
   bool make_empty_co_sstable = false;
+  create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
   ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
-      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, make_empty_co_sstable, &freezer);
+      create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
   ASSERT_EQ(common::OB_SUCCESS, ret);
   ASSERT_EQ(1, tablet->get_ref());
 

src/objit/include/objit/common/ob_item_type.h

@@ -1128,6 +1128,7 @@ typedef enum ObItemType
   T_ALTER_PARTITION_DROP,
   T_ALTER_PARTITION_ADD,
   T_ALTER_COLUMN_OPTION,
+  T_ALTER_COLUMN_GROUP_OPTION,
   T_ALTER_TABLEGROUP_OPTION,
   T_ALTER_TABLE_ACTION_LIST,
 
@@ -1141,6 +1142,8 @@ typedef enum ObItemType
   T_COLUMN_RENAME,
   T_COLUMN_MODIFY,
   T_COLUMN_CHANGE,
+  T_COLUMN_GROUP_ADD,
+  T_COLUMN_GROUP_DROP,
   T_INDEX_ADD,
   T_INDEX_DROP,
   T_INDEX_ALTER,

src/observer/ob_rpc_processor_simple.cpp

@@ -45,6 +45,7 @@
 // for 4.0
 #include "share/ob_ls_id.h"
 #include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
+#include "storage/ddl/ob_direct_insert_sstable_ctx_new.h"
 #include "storage/ls/ob_ls.h"
 #include "storage/tablet/ob_tablet.h"
 #include "storage/tx/ob_trans_service.h"
@@ -2319,14 +2320,12 @@ int ObRpcRemoteWriteDDLRedoLogP::process()
   } else {
     MTL_SWITCH(tenant_id) {
       ObRole role = INVALID_ROLE;
-      ObDDLSSTableRedoWriter sstable_redo_writer;
+      ObDDLRedoLogWriter sstable_redo_writer;
       MacroBlockId macro_block_id;
       ObMacroBlockHandle macro_handle;
       ObMacroBlockWriteInfo write_info;
       ObLSService *ls_service = MTL(ObLSService*);
       ObLSHandle ls_handle;
-      ObTabletHandle tablet_handle;
-      ObDDLKvMgrHandle ddl_kv_mgr_handle;
       ObLS *ls = nullptr;
 
       // restruct write_info
@@ -2344,19 +2343,15 @@ int ObRpcRemoteWriteDDLRedoLogP::process()
       } else if (ObRole::LEADER != role) {
         ret = OB_NOT_MASTER;
         LOG_INFO("leader may not have finished replaying clog, caller retry", K(ret), K(MTL_ID()), K(arg_.ls_id_));
-      } else if (OB_FAIL(ls->get_tablet(arg_.redo_info_.table_key_.tablet_id_, tablet_handle))) {
-        LOG_WARN("get tablet failed", K(ret));
-      } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
-        LOG_WARN("get ddl kv manager failed", K(ret));
       } else if (OB_FAIL(ObBlockManager::async_write_block(write_info, macro_handle))) {
         LOG_WARN("fail to async write block", K(ret), K(write_info), K(macro_handle));
       } else if (OB_FAIL(macro_handle.wait())) {
         LOG_WARN("fail to wait macro block io finish", K(ret));
       } else if (OB_FAIL(sstable_redo_writer.init(arg_.ls_id_, arg_.redo_info_.table_key_.tablet_id_))) {
         LOG_WARN("init sstable redo writer", K(ret), K_(arg));
-      } else if (OB_FAIL(sstable_redo_writer.write_redo_log(arg_.redo_info_, macro_handle.get_macro_id(), false, arg_.task_id_, tablet_handle, ddl_kv_mgr_handle))) {
+      } else if (OB_FAIL(sstable_redo_writer.write_macro_block_log(arg_.redo_info_, macro_handle.get_macro_id(), false, arg_.task_id_))) {
         LOG_WARN("fail to write macro redo", K(ret), K_(arg));
-      } else if (OB_FAIL(sstable_redo_writer.wait_redo_log_finish(arg_.redo_info_,
+      } else if (OB_FAIL(sstable_redo_writer.wait_macro_block_log_finish(arg_.redo_info_,
                                                                   macro_handle.get_macro_id()))) {
         LOG_WARN("fail to wait macro redo finish", K(ret), K_(arg));
       }
@@ -2373,12 +2368,15 @@ int ObRpcRemoteWriteDDLCommitLogP::process()
   MTL_SWITCH(tenant_id) {
     ObRole role = INVALID_ROLE;
     const ObITable::TableKey &table_key = arg_.table_key_;
-    ObDDLSSTableRedoWriter sstable_redo_writer;
+    ObDDLRedoLogWriter sstable_redo_writer;
     ObLSService *ls_service = MTL(ObLSService*);
     ObLSHandle ls_handle;
-    ObTabletHandle tablet_handle;
-    ObDDLKvMgrHandle ddl_kv_mgr_handle;
     ObLS *ls = nullptr;
+    ObTenantDirectLoadMgr *tenant_direct_load_mgr = MTL(ObTenantDirectLoadMgr *);
+    ObTabletFullDirectLoadMgr *data_tablet_mgr = nullptr;
+    ObTabletDirectLoadMgrHandle direct_load_mgr_handle;
+    direct_load_mgr_handle.reset();
+    bool is_major_sstable_exist = false;
     if (OB_UNLIKELY(!arg_.is_valid())) {
       ret = OB_INVALID_ARGUMENT;
       LOG_WARN("invalid arguments", K(ret), K_(arg));
@@ -2392,32 +2390,55 @@ int ObRpcRemoteWriteDDLCommitLogP::process()
     } else if (ObRole::LEADER != role) {
       ret = OB_NOT_MASTER;
       LOG_INFO("leader may not have finished replaying clog, caller retry", K(ret), K(MTL_ID()), K(arg_.ls_id_));
-    } else if (OB_FAIL(ls->get_tablet(table_key.tablet_id_, tablet_handle))) {
-      LOG_WARN("get tablet failed", K(ret));
-    } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
-      if (OB_ENTRY_NOT_EXIST == ret) {
-        ret = OB_EAGAIN;
+    } else if (OB_ISNULL(tenant_direct_load_mgr)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("unexpected nullptr", K(ret), K(MTL_ID()));
+    } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major(
+            arg_.ls_id_,
+            table_key.tablet_id_,
+            true /*is_full_direct_load*/,
+            direct_load_mgr_handle,
+            is_major_sstable_exist))) {
+      if (OB_ENTRY_NOT_EXIST == ret && is_major_sstable_exist) {
+        ret = OB_TASK_EXPIRED;
+        LOG_INFO("major sstable already exist", K(ret), K(arg_));
       } else {
-        LOG_WARN("get ddl kv manager failed", K(ret));
+        LOG_WARN("get tablet direct load manager failed", K(ret), K(table_key));
       }
+    } else if (OB_ISNULL(data_tablet_mgr = direct_load_mgr_handle.get_full_obj())) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("unexpected err", K(ret), K(table_key));
     } else if (OB_FAIL(sstable_redo_writer.init(arg_.ls_id_, table_key.tablet_id_))) {
       LOG_WARN("init sstable redo writer", K(ret), K(table_key));
-    } else if (FALSE_IT(sstable_redo_writer.set_start_scn(arg_.start_scn_))) {
     } else {
+      uint32_t lock_tid = 0;
       SCN commit_scn;
       bool is_remote_write = false;
-      if (OB_FAIL(sstable_redo_writer.write_commit_log(tablet_handle,
-                                                       ddl_kv_mgr_handle,
-                                                       false,
-                                                       table_key,
-                                                       commit_scn,
-                                                       is_remote_write))) {
+      ObTabletHandle tablet_handle;
+      if (OB_FAIL(data_tablet_mgr->wrlock(ObTabletDirectLoadMgr::TRY_LOCK_TIMEOUT, lock_tid))) {
+        LOG_WARN("failed to wrlock", K(ret), K(arg_));
+      } else if (OB_FAIL(sstable_redo_writer.write_commit_log(false,
+                                                              table_key,
+                                                              arg_.start_scn_,
+                                                              direct_load_mgr_handle,
+                                                              commit_scn,
+                                                              is_remote_write,
+                                                              lock_tid))) {
         LOG_WARN("fail to remote write commit log", K(ret), K(table_key), K_(arg));
-      } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_commit(*tablet_handle.get_obj(), arg_.start_scn_, commit_scn))) {
+      } else if (OB_FAIL(ls->get_tablet(table_key.tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) {
+        LOG_WARN("get tablet failed", K(ret), K(table_key));
+      } else if (OB_FAIL(data_tablet_mgr->commit(*tablet_handle.get_obj(),
+                                                 arg_.start_scn_,
+                                                 commit_scn,
+                                                 arg_.table_id_,
+                                                 arg_.ddl_task_id_))) {
         LOG_WARN("failed to do ddl kv commit", K(ret), K(arg_));
       } else {
         result_ = commit_scn.get_val_for_tx();
       }
+      if (lock_tid != 0) {
+        data_tablet_mgr->unlock(lock_tid);
+      }
     }
   }
   return ret;

src/observer/ob_server.cpp

@@ -88,7 +88,6 @@
 #include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h"
 #include "storage/tx_storage/ob_tenant_freezer.h"
 #include "storage/tx_storage/ob_tenant_memory_printer.h"
-#include "storage/ddl/ob_direct_insert_sstable_ctx.h"
 #include "storage/compaction/ob_compaction_diagnose.h"
 #include "storage/ob_file_system_router.h"
 #include "storage/blocksstable/ob_storage_cache_suite.h"
@@ -482,12 +481,12 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
       LOG_ERROR("init server blacklist failed", KR(ret));
     } else if (OB_FAIL(ObLongopsMgr::get_instance().init())) {
       LOG_WARN("init longops mgr fail", KR(ret));
+    } else if (OB_FAIL(ObDDLRedoLock::get_instance().init())) {
+      LOG_WARN("init ddl redo lock failed", K(ret));
 #ifdef ERRSIM
     } else if (OB_FAIL(ObDDLSimPointMgr::get_instance().init())) {
       LOG_WARN("init ddl sim point mgr fail", KR(ret));
 #endif
-    } else if (OB_FAIL(ObDDLRedoLogWriter::get_instance().init())) {
-      LOG_WARN("init DDL redo log writer failed", KR(ret));
     }
 #ifdef OB_BUILD_ARBITRATION
     else if (OB_FAIL(arb_gcs_.init(GCTX.self_addr(),
@@ -2708,12 +2707,6 @@ int ObServer::init_storage()
     }
   }
 
-  if (OB_SUCC(ret)) {
-    if (OB_FAIL(ObSSTableInsertManager::get_instance().init())) {
-      LOG_WARN("init direct insert sstable manager failed", KR(ret));
-    }
-  }
-
   if (OB_SUCC(ret)) {
     if (OB_FAIL(ObDDLCtrlSpeedHandle::get_instance().init())) {
       LOG_WARN("fail to init ObDDLCtrlSpeedHandle", KR(ret));

src/observer/omt/ob_multi_tenant.cpp

@@ -53,6 +53,7 @@
 #include "logservice/data_dictionary/ob_data_dict_service.h" // ObDataDictService
 #include "ob_tenant_mtl_helper.h"
 #include "storage/blocksstable/ob_decode_resource_pool.h"
+#include "storage/ddl/ob_direct_insert_sstable_ctx_new.h"
 #include "storage/multi_data_source/runtime_utility/mds_tenant_service.h"
 #include "storage/tx_storage/ob_ls_service.h"
 #include "storage/tx_storage/ob_access_service.h"
@@ -540,6 +541,7 @@ int ObMultiTenant::init(ObAddr myaddr,
     MTL_BIND2(mtl_new_default, ObPsCache::mtl_init, nullptr, ObPsCache::mtl_stop, nullptr, mtl_destroy_default);
     MTL_BIND2(server_obj_pool_mtl_new<ObPartTransCtx>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<ObPartTransCtx>);
     MTL_BIND2(server_obj_pool_mtl_new<ObTableScanIterator>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<ObTableScanIterator>);
+    MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
     MTL_BIND(ObDetectManager::mtl_init, ObDetectManager::mtl_destroy);
     MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy);
     MTL_BIND2(mtl_new_default, ObDTLIntermResultManager::mtl_init, ObDTLIntermResultManager::mtl_start,

src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp

@@ -218,13 +218,10 @@ int ObTableDirectLoadBeginExecutor::create_table_ctx()
   if (OB_SUCC(ret)) {
     ObTableLoadRedefTableStartArg start_arg;
     ObTableLoadRedefTableStartRes start_res;
-    uint64_t data_version = 0;
     start_arg.tenant_id_ = tenant_id;
     start_arg.table_id_ = table_id;
     start_arg.parallelism_ = arg_.parallel_;
-    if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) {
-      LOG_WARN("fail to get tenant data version", KR(ret));
-    } else if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res,
+    if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res,
                                                     *client_task_->get_session_info()))) {
       LOG_WARN("fail to start redef table", KR(ret), K(start_arg));
     } else {
@@ -232,7 +229,7 @@ int ObTableDirectLoadBeginExecutor::create_table_ctx()
       ddl_param.task_id_ = start_res.task_id_;
       ddl_param.schema_version_ = start_res.schema_version_;
       ddl_param.snapshot_version_ = start_res.snapshot_version_;
-      ddl_param.data_version_ = data_version;
+      ddl_param.data_version_ = start_res.data_format_version_;
     }
   }
   // init param

src/observer/table_load/ob_table_load_instance.cpp

@@ -114,21 +114,18 @@ int ObTableLoadInstance::create_table_ctx(ObTableLoadParam &param,
   // start redef table
   ObTableLoadRedefTableStartArg start_arg;
   ObTableLoadRedefTableStartRes start_res;
-  uint64_t data_version = 0;
   start_arg.tenant_id_ = param.tenant_id_;
   start_arg.table_id_ = param.table_id_;
   start_arg.parallelism_ = param.parallel_;
   start_arg.is_load_data_ = !param.px_mode_;
-  if (OB_FAIL(GET_MIN_DATA_VERSION(param.tenant_id_, data_version))) {
-    LOG_WARN("fail to get tenant data version", KR(ret));
-  } else if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *session_info))) {
+  if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *session_info))) {
     LOG_WARN("fail to start redef table", KR(ret), K(start_arg));
   } else {
     ddl_param.dest_table_id_ = start_res.dest_table_id_;
     ddl_param.task_id_ = start_res.task_id_;
     ddl_param.schema_version_ = start_res.schema_version_;
     ddl_param.snapshot_version_ = start_res.snapshot_version_;
-    ddl_param.data_version_ = data_version;
+    ddl_param.data_version_ = start_res.data_format_version_;
   }
   if (OB_SUCC(ret)) {
     if (OB_ISNULL(table_ctx = ObTableLoadService::alloc_ctx())) {

src/observer/table_load/ob_table_load_merger.cpp

@@ -106,6 +106,74 @@ private:
   ObTableLoadMerger *const merger_;
 };
 
+class ObTableLoadMerger::RescanTaskProcessor : public ObITableLoadTaskProcessor
+{
+public:
+  RescanTaskProcessor(ObTableLoadTask &task, ObTableLoadTableCtx *ctx, ObTableLoadMerger *merger)
+    : ObITableLoadTaskProcessor(task), ctx_(ctx), merger_(merger)
+  {
+    ctx_->inc_ref_count();
+  }
+  virtual ~RescanTaskProcessor()
+  {
+    ObTableLoadService::put_ctx(ctx_);
+  }
+  int process() override
+  {
+    int ret = OB_SUCCESS;
+    ObDirectLoadPartitionRescanTask *rescan_task = nullptr;
+    while (OB_SUCC(ret)) {
+      rescan_task = nullptr;
+      if (OB_FAIL(merger_->get_next_rescan_task(rescan_task))) {
+        if (OB_UNLIKELY(OB_ITER_END != ret)) {
+          LOG_WARN("fail to get next rescan task", KR(ret));
+        } else {
+          ret = OB_SUCCESS;
+          break;
+        }
+      } else if (OB_FAIL(rescan_task->process())) {
+        LOG_WARN("fail to process rescan task", KR(ret));
+      }
+      if (nullptr != rescan_task) {
+        merger_->handle_rescan_task_finish(rescan_task);
+      }
+    }
+    return ret;
+  }
+private:
+  ObTableLoadTableCtx *const ctx_;
+  ObTableLoadMerger *const merger_;
+};
+
+class ObTableLoadMerger::RescanTaskCallback : public ObITableLoadTaskCallback
+{
+public:
+  RescanTaskCallback(ObTableLoadTableCtx *ctx, ObTableLoadMerger *merger)
+    : ctx_(ctx), merger_(merger)
+  {
+    ctx_->inc_ref_count();
+  }
+  virtual ~RescanTaskCallback()
+  {
+    ObTableLoadService::put_ctx(ctx_);
+  }
+  void callback(int ret_code, ObTableLoadTask *task) override
+  {
+    int ret = OB_SUCCESS;
+    if (OB_FAIL(merger_->handle_rescan_thread_finish(ret_code))) {
+      LOG_WARN("fail to handle rescan thread finish", KR(ret));
+    }
+    if (OB_FAIL(ret)) {
+      ctx_->store_ctx_->set_status_error(ret);
+    }
+    ctx_->free_task(task);
+    OB_TABLE_LOAD_STATISTICS_PRINT_AND_RESET();
+  }
+private:
+  ObTableLoadTableCtx *const ctx_;
+  ObTableLoadMerger *const merger_;
+};
+
 /**
  * ObTableLoadMerger
  */
@@ -123,6 +191,7 @@ ObTableLoadMerger::ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx)
 ObTableLoadMerger::~ObTableLoadMerger()
 {
   abort_unless(merging_list_.is_empty());
+  abort_unless(rescan_list_.is_empty());
 }
 
 int ObTableLoadMerger::init()
@@ -172,9 +241,13 @@ void ObTableLoadMerger::stop()
   // 遍历合并中的任务队列, 调用stop
   ObMutexGuard guard(mutex_);
   ObDirectLoadPartitionMergeTask *merge_task = nullptr;
+  ObDirectLoadPartitionRescanTask *rescan_task = nullptr;
   DLIST_FOREACH_NORET(merge_task, merging_list_) {
     merge_task->stop();
   }
+  DLIST_FOREACH_NORET(rescan_task, rescan_list_) {
+    rescan_task->stop();
+  }
 }
 
 int ObTableLoadMerger::handle_table_compact_success()
@@ -203,10 +276,13 @@ int ObTableLoadMerger::build_merge_ctx()
   merge_param.table_data_desc_ = store_ctx_->table_data_desc_;
   merge_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_);
   merge_param.col_descs_ = &(store_ctx_->ctx_->schema_.column_descs_);
+  merge_param.lob_column_cnt_ = store_ctx_->ctx_->schema_.lob_column_cnt_;
   merge_param.cmp_funcs_ = &(store_ctx_->ctx_->schema_.cmp_funcs_);
   merge_param.is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_;
   merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_;
   merge_param.online_opt_stat_gather_ = param_.online_opt_stat_gather_;
+  merge_param.is_column_store_ = store_ctx_->ctx_->schema_.is_column_store_;
+  merge_param.px_mode_ = param_.px_mode_;
   merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_;
   merge_param.dml_row_handler_ = store_ctx_->error_row_handler_;
   if (OB_FAIL(merge_ctx_.init(merge_param, store_ctx_->ls_partition_ids_,
@@ -417,11 +493,37 @@ int ObTableLoadMerger::collect_sql_statistics(ObTableLoadSqlStatistics &sql_stat
   return ret;
 }
 
+int ObTableLoadMerger::build_rescan_ctx()
+{
+  int ret = OB_SUCCESS;
+  const ObIArray<ObDirectLoadTabletMergeCtx *> &tablet_merge_ctxs =
+    merge_ctx_.get_tablet_merge_ctxs();
+  for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) {
+    ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i);
+    if (OB_FAIL(tablet_merge_ctx->build_rescan_task(param_.session_count_))) {
+      LOG_WARN("fail to build rescan task", KR(ret));
+    }
+  }
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(rescan_task_iter_.init(&merge_ctx_))) {
+      LOG_WARN("fail to build rescan task", KR(ret));
+    }
+  }
+  return ret;
+
+}
+
 int ObTableLoadMerger::start_merge()
 {
   int ret = OB_SUCCESS;
   const int64_t thread_count = store_ctx_->task_scheduler_->get_thread_count();
   ObTableLoadTableCtx *ctx = store_ctx_->ctx_;
+  if (OB_UNLIKELY(0 != running_thread_count_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpected running thread count not zero", KR(ret), K(running_thread_count_));
+  } else {
+    running_thread_count_ = thread_count;
+  }
   for (int32_t thread_idx = 0; OB_SUCC(ret) && thread_idx < thread_count; ++thread_idx) {
     ObTableLoadTask *task = nullptr;
     // 1. 分配task
@@ -440,9 +542,46 @@ int ObTableLoadMerger::start_merge()
     else if (OB_FAIL(store_ctx_->task_scheduler_->add_task(thread_idx, task))) {
       LOG_WARN("fail to add task", KR(ret), K(thread_idx), KPC(task));
     }
-    // 5. inc running_thread_count_
-    else {
-      ATOMIC_INC(&running_thread_count_);
+    if (OB_FAIL(ret)) {
+      if (nullptr != task) {
+        ctx->free_task(task);
+      }
+    }
+  }
+  if (OB_FAIL(ret)) {
+    has_error_ = true;
+  }
+  return ret;
+}
+
+int ObTableLoadMerger::start_rescan()
+{
+  int ret = OB_SUCCESS;
+  const int64_t thread_count = store_ctx_->task_scheduler_->get_thread_count();
+  ObTableLoadTableCtx *ctx = store_ctx_->ctx_;
+  if (OB_UNLIKELY(0 != running_thread_count_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpected running thread count not zero", KR(ret), K(running_thread_count_));
+  } else {
+    running_thread_count_ = thread_count;
+  }
+  for (int32_t thread_idx = 0; OB_SUCC(ret) && thread_idx < thread_count; ++thread_idx) {
+    ObTableLoadTask *task = nullptr;
+    // 1. 分配task
+    if (OB_FAIL(ctx->alloc_task(task))) {
+      LOG_WARN("fail to alloc task", KR(ret));
+    }
+    // 2. 设置processor
+    else if (OB_FAIL(task->set_processor<RescanTaskProcessor>(ctx, this))) {
+      LOG_WARN("fail to set merge task processor", KR(ret));
+    }
+    // 3. 设置callback
+    else if (OB_FAIL(task->set_callback<RescanTaskCallback>(ctx, this))) {
+      LOG_WARN("fail to set merge task callback", KR(ret));
+    }
+    // 4. 把task放入调度器
+    else if (OB_FAIL(store_ctx_->task_scheduler_->add_task(thread_idx, task))) {
+      LOG_WARN("fail to add task", KR(ret), K(thread_idx), KPC(task));
     }
     if (OB_FAIL(ret)) {
       if (nullptr != task) {
@@ -456,6 +595,25 @@ int ObTableLoadMerger::start_merge()
   return ret;
 }
 
+int ObTableLoadMerger::get_next_rescan_task(ObDirectLoadPartitionRescanTask *&rescan_task)
+{
+  int ret = OB_SUCCESS;
+  rescan_task = nullptr;
+  if (OB_UNLIKELY(is_stop_ || has_error_)) {
+    ret = OB_ITER_END;
+  } else {
+    ObMutexGuard guard(mutex_);
+    if (OB_FAIL(rescan_task_iter_.get_next_task(rescan_task))) {
+      if (OB_UNLIKELY(OB_ITER_END != ret)) {
+        LOG_WARN("fail to get next task", KR(ret));
+      }
+    } else {
+      OB_ASSERT(rescan_list_.add_last(rescan_task));
+    }
+  }
+  return ret;
+}
+
 int ObTableLoadMerger::get_next_merge_task(ObDirectLoadPartitionMergeTask *&merge_task)
 {
   int ret = OB_SUCCESS;
@@ -481,6 +639,12 @@ void ObTableLoadMerger::handle_merge_task_finish(ObDirectLoadPartitionMergeTask
   OB_ASSERT(OB_NOT_NULL(merging_list_.remove(merge_task)));
 }
 
+void ObTableLoadMerger::handle_rescan_task_finish(ObDirectLoadPartitionRescanTask *&rescan_task)
+{
+  ObMutexGuard guard(mutex_);
+  OB_ASSERT(OB_NOT_NULL(rescan_list_.remove(rescan_task)));
+}
+
 int ObTableLoadMerger::handle_merge_thread_finish(int ret_code)
 {
   int ret = OB_SUCCESS;
@@ -492,6 +656,33 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code)
     if (OB_UNLIKELY(is_stop_ || has_error_)) {
     } else {
       LOG_INFO("LOAD MERGE COMPLETED");
+      if (store_ctx_->ctx_->schema_.is_column_store_) {
+        if (OB_FAIL(build_rescan_ctx())) {
+          LOG_WARN("fail to build rescan ctx", KR(ret));
+        } else if (OB_FAIL(start_rescan())) {
+          LOG_WARN("fail to start rescan", KR(ret));
+        }
+      } else {
+        if (OB_FAIL(store_ctx_->set_status_merged())) {
+          LOG_WARN("fail to set store status merged", KR(ret));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObTableLoadMerger::handle_rescan_thread_finish(const int ret_code)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(ret_code)) {
+    has_error_ = true;
+  }
+  const int64_t running_thread_count = ATOMIC_SAF(&running_thread_count_, 1);
+  if (0 == running_thread_count) {
+    if (OB_UNLIKELY(is_stop_ || has_error_)) {
+    } else {
+      LOG_INFO("LOAD RESCAN COMPLETED");
       if (OB_FAIL(store_ctx_->set_status_merged())) {
         LOG_WARN("fail to set store status merged", KR(ret));
       }

src/observer/table_load/ob_table_load_merger.h

@@ -18,6 +18,7 @@
 #include "share/table/ob_table_load_define.h"
 #include "storage/direct_load/ob_direct_load_merge_ctx.h"
 #include "storage/direct_load/ob_direct_load_merge_task_iterator.h"
+#include "storage/direct_load/ob_direct_load_partition_rescan_task.h"
 #include "storage/direct_load/ob_direct_load_partition_merge_task.h"
 
 namespace oceanbase
@@ -30,7 +31,9 @@ class ObTableLoadStoreCtx;
 class ObTableLoadMerger
 {
   class MergeTaskProcessor;
+  class RescanTaskProcessor;
   class MergeTaskCallback;
+  class RescanTaskCallback;
 public:
   ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx);
   ~ObTableLoadMerger();
@@ -42,10 +45,15 @@ public:
   int collect_dml_stat(table::ObTableLoadDmlStat &dml_stats);
 private:
   int build_merge_ctx();
+  int build_rescan_ctx();
   int start_merge();
+  int start_rescan();
   int get_next_merge_task(storage::ObDirectLoadPartitionMergeTask *&merge_task);
+  int get_next_rescan_task(ObDirectLoadPartitionRescanTask *&rescan_task);
   void handle_merge_task_finish(storage::ObDirectLoadPartitionMergeTask *&merge_task);
   int handle_merge_thread_finish(int ret_code);
+  void handle_rescan_task_finish(ObDirectLoadPartitionRescanTask *&rescan_task);
+  int handle_rescan_thread_finish(const int ret_code);
 private:
   ObTableLoadStoreCtx * const store_ctx_;
   const ObTableLoadParam &param_;
@@ -53,7 +61,9 @@ private:
   storage::ObDirectLoadMergeCtx merge_ctx_;
   mutable lib::ObMutex mutex_;
   ObDirectLoadMergeTaskIterator merge_task_iter_;
+  ObDirectLoadRescanTaskIterator rescan_task_iter_;
   common::ObDList<storage::ObDirectLoadPartitionMergeTask> merging_list_;
+  common::ObDList<storage::ObDirectLoadPartitionRescanTask> rescan_list_;
   int64_t running_thread_count_ CACHE_ALIGNED;
   volatile bool has_error_;
   volatile bool is_stop_;

src/observer/table_load/ob_table_load_redef_table.cpp

@@ -38,7 +38,6 @@ int ObTableLoadRedefTable::start(const ObTableLoadRedefTableStartArg &arg,
     const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts();
     ObCreateHiddenTableArg create_table_arg;
     ObCreateHiddenTableRes create_table_res;
-    int64_t snapshot_version = OB_INVALID_VERSION;
     create_table_arg.reset();
     create_table_arg.exec_tenant_id_ = arg.tenant_id_;
     create_table_arg.tenant_id_ = arg.tenant_id_;
@@ -55,16 +54,16 @@ int ObTableLoadRedefTable::start(const ObTableLoadRedefTableStartArg &arg,
     create_table_arg.consumer_group_id_ = THIS_WORKER.get_group_id();
     if (OB_FAIL(create_table_arg.tz_info_wrap_.deep_copy(session_info.get_tz_info_wrap()))) {
       LOG_WARN("failed to deep copy tz_info_wrap", KR(ret));
-    } else if (OB_FAIL(ObDDLServerClient::create_hidden_table(create_table_arg, create_table_res, snapshot_version, session_info))) {
+    } else if (OB_FAIL(ObDDLServerClient::create_hidden_table(create_table_arg, create_table_res,
+        res.snapshot_version_, res.data_format_version_, session_info))) {
       LOG_WARN("failed to create hidden table", KR(ret), K(create_table_arg));
-    } else if (OB_UNLIKELY(snapshot_version <= 0)) {
+    } else if (OB_UNLIKELY(res.snapshot_version_ <= 0 || res.data_format_version_ <= 0)) {
       ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("invalid snapshot version", K(ret));
+      LOG_WARN("invalid snapshot version", K(ret), K(res));
     } else {
       res.dest_table_id_ = create_table_res.dest_table_id_;
       res.task_id_ = create_table_res.task_id_;
       res.schema_version_ = create_table_res.schema_version_;
-      res.snapshot_version_ = snapshot_version;
       LOG_INFO("succeed to create hidden table", K(arg), K(res));
     }
     THIS_WORKER.set_timeout_ts(origin_timeout_ts);

src/observer/table_load/ob_table_load_redef_table.h

@@ -57,7 +57,7 @@ struct ObTableLoadRedefTableStartRes
 {
 public:
   ObTableLoadRedefTableStartRes()
-    : dest_table_id_(common::OB_INVALID_ID), task_id_(0), schema_version_(0), snapshot_version_(0)
+    : dest_table_id_(common::OB_INVALID_ID), task_id_(0), schema_version_(0), snapshot_version_(0), data_format_version_(0)
   {
   }
   ~ObTableLoadRedefTableStartRes() = default;
@@ -67,13 +67,15 @@ public:
     task_id_ = 0;
     schema_version_ = 0;
     snapshot_version_ = 0;
+    data_format_version_ = 0;
   }
-  TO_STRING_KV(K_(dest_table_id), K_(task_id), K_(schema_version), K_(snapshot_version));
+  TO_STRING_KV(K_(dest_table_id), K_(task_id), K_(schema_version), K_(snapshot_version), K_(data_format_version));
 public:
   uint64_t dest_table_id_;
   int64_t task_id_;
   int64_t schema_version_;
   int64_t snapshot_version_;
+  uint64_t data_format_version_;
 };
 
 struct ObTableLoadRedefTableFinishArg

src/observer/table_load/ob_table_load_schema.cpp

@@ -195,10 +195,12 @@ ObTableLoadSchema::ObTableLoadSchema()
   : allocator_("TLD_Schema"),
     is_partitioned_table_(false),
     is_heap_table_(false),
+    is_column_store_(false),
     has_autoinc_column_(false),
     has_identity_column_(false),
     rowkey_column_count_(0),
     store_column_count_(0),
+    lob_column_cnt_(0),
     collation_type_(CS_TYPE_INVALID),
     schema_version_(0),
     is_inited_(false)
@@ -217,10 +219,12 @@ void ObTableLoadSchema::reset()
   table_name_.reset();
   is_partitioned_table_ = false;
   is_heap_table_ = false;
+  is_column_store_ = false;
   has_autoinc_column_ = false;
   has_identity_column_ = false;
   rowkey_column_count_ = 0;
   store_column_count_ = 0;
+  lob_column_cnt_ = 0;
   collation_type_ = CS_TYPE_INVALID;
   schema_version_ = 0;
   column_descs_.reset();
@@ -262,6 +266,7 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema)
   } else {
     is_partitioned_table_ = table_schema->is_partitioned_table();
     is_heap_table_ = table_schema->is_heap_table();
+    is_column_store_ = (table_schema->get_column_group_count() > 1) ? true :false;
     has_autoinc_column_ = (table_schema->get_autoinc_column_id() != 0);
     rowkey_column_count_ = table_schema->get_rowkey_column_num();
     collation_type_ = table_schema->get_collation_type();
@@ -282,6 +287,8 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema)
     } else if (OB_FAIL(datum_utils_.init(multi_version_column_descs_, rowkey_column_count_,
                                          lib::is_oracle_mode(), allocator_))) {
       LOG_WARN("fail to init datum utils", KR(ret));
+    } else if (OB_FAIL(init_lob_storage(column_descs_))) {
+      LOG_WARN("fail to check lob storage", KR(ret));
     } else if (OB_FAIL(init_cmp_funcs(column_descs_, lib::is_oracle_mode()))) {
       LOG_WARN("fail to init cmp funcs", KR(ret));
     }
@@ -317,6 +324,20 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema)
   return ret;
 }
 
+int ObTableLoadSchema::init_lob_storage(common::ObIArray<share::schema::ObColDesc> &column_descs)
+{
+  int ret = OB_SUCCESS;
+  lob_column_cnt_ = 0;
+  for (int64_t i = 0; OB_SUCC(ret) && i < column_descs.count(); ++i) {
+    const ObColDesc &col_desc = column_descs.at(i);
+    if (col_desc.col_type_.is_lob_storage()) {
+      column_descs.at(i).col_type_.set_has_lob_header();
+      ++lob_column_cnt_;
+    }
+  }
+  return ret;
+}
+
 int ObTableLoadSchema::init_cmp_funcs(const ObIArray<ObColDesc> &col_descs,
                                       const bool is_oracle_mode)
 {

src/observer/table_load/ob_table_load_schema.h

@@ -55,22 +55,25 @@ public:
                K_(collation_type), K_(column_descs), K_(is_inited));
 private:
   int init_table_schema(const share::schema::ObTableSchema *table_schema);
+  int init_cmp_funcs(const common::ObIArray<share::schema::ObColDesc> &column_descs,
+                     const bool is_oracle_mode);
+  int init_lob_storage(common::ObIArray<share::schema::ObColDesc> &column_descs);
   int update_decimal_int_precision(const share::schema::ObTableSchema *table_schema,
                                    common::ObIArray<share::schema::ObColDesc> &cols_desc);
 
   int prepare_col_desc(const ObTableSchema *table_schema, common::ObIArray<share::schema::ObColDesc> &col_descs);
-  int init_cmp_funcs(const common::ObIArray<share::schema::ObColDesc> &column_descs,
-                     const bool is_oracle_mode);
 public:
   common::ObArenaAllocator allocator_;
   common::ObString table_name_;
   bool is_partitioned_table_;
   bool is_heap_table_;
+  bool is_column_store_;
   bool has_autoinc_column_;
   bool has_identity_column_;
   int64_t rowkey_column_count_;
   // column count in store, does not contain virtual generated columns
   int64_t store_column_count_;
+  int64_t lob_column_cnt_;
   common::ObCollationType collation_type_;
   int64_t schema_version_;
   // if it is a heap table, it contains hidden primary key column

src/observer/table_load/ob_table_load_store.cpp

@@ -311,8 +311,6 @@ int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info)
     ObTableLoadSqlStatistics sql_statistics;
     if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
       LOG_WARN("fail to check store status", KR(ret));
-    } else if (OB_FAIL(store_ctx_->insert_table_ctx_->commit())) {
-      LOG_WARN("fail to commit insert table", KR(ret));
     } else if (ctx_->schema_.has_autoinc_column_ && OB_FAIL(store_ctx_->commit_autoinc_value())) {
       LOG_WARN("fail to commit sync auto increment value", KR(ret));
     } else if (param_.online_opt_stat_gather_ &&

src/observer/table_load/ob_table_load_store_ctx.cpp

@@ -24,7 +24,6 @@
 #include "share/sequence/ob_sequence_cache.h"
 #include "sql/engine/cmd/ob_load_data_utils.h"
 #include "storage/direct_load/ob_direct_load_data_block.h"
-#include "storage/direct_load/ob_direct_load_fast_heap_table_ctx.h"
 #include "storage/direct_load/ob_direct_load_insert_table_ctx.h"
 #include "storage/direct_load/ob_direct_load_mem_context.h"
 #include "storage/direct_load/ob_direct_load_sstable_data_block.h"
@@ -51,7 +50,6 @@ ObTableLoadStoreCtx::ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx)
     insert_table_ctx_(nullptr),
     is_multiple_mode_(false),
     is_fast_heap_table_(false),
-    fast_heap_table_ctx_(nullptr),
     tmp_file_mgr_(nullptr),
     error_row_handler_(nullptr),
     sequence_schema_(&allocator_),
@@ -88,6 +86,7 @@ int ObTableLoadStoreCtx::init(
     insert_table_param.ddl_task_id_ = ctx_->ddl_param_.task_id_;
     insert_table_param.execution_id_ = 1; //仓氐说暂时设置为1，不然后面检测过不了
     insert_table_param.data_version_ = ctx_->ddl_param_.data_version_;
+    insert_table_param.reserved_parallel_ = ctx_->param_.session_count_;
     for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) {
       const ObLSID &ls_id = partition_id_array[i].ls_id_;
       const ObTableLoadPartitionId &part_tablet_id = partition_id_array[i].part_tablet_id_;
@@ -138,7 +137,12 @@ int ObTableLoadStoreCtx::init(
       }
       if (OB_SUCC(ret)) {
         if (table_data_desc_.is_heap_table_) {
-          int64_t bucket_cnt = wa_mem_limit / (ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE);
+          int64_t bucket_cnt = 0;
+          if (ctx_->schema_.lob_column_cnt_ > 0) {
+            bucket_cnt = wa_mem_limit / (ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE * 2);
+          } else {
+            bucket_cnt = wa_mem_limit / (ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE);
+          }
           if (ls_partition_ids_.count() <= bucket_cnt) {
             is_fast_heap_table_ = true;
           } else {
@@ -154,7 +158,9 @@ int ObTableLoadStoreCtx::init(
       }
     }
     if (OB_FAIL(ret)) {
-    } else if (OB_FAIL(insert_table_param.ls_partition_ids_.assign(target_ls_partition_ids_))) {
+    } else if (OB_FAIL(insert_table_param.ls_partition_ids_.assign(ls_partition_ids_))) {
+      LOG_WARN("fail to assign ls tablet ids", KR(ret));
+    } else if (OB_FAIL(insert_table_param.target_ls_partition_ids_.assign(target_ls_partition_ids_))) {
       LOG_WARN("fail to assign ls tablet ids", KR(ret));
     }
     // init trans_allocator_
@@ -223,17 +229,6 @@ int ObTableLoadStoreCtx::init(
     else if (ctx_->schema_.has_identity_column_ && OB_FAIL(init_sequence())) {
       LOG_WARN("fail to init sequence", KR(ret));
     }
-    if (OB_SUCC(ret) && is_fast_heap_table_) {
-      if (OB_ISNULL(fast_heap_table_ctx_ =
-                      OB_NEWx(ObDirectLoadFastHeapTableContext, (&allocator_)))) {
-        ret = OB_ALLOCATE_MEMORY_FAILED;
-        LOG_WARN("fail to new ObDirectLoadFastHeapTableContext", KR(ret));
-      } else if (OB_FAIL(fast_heap_table_ctx_->init(ctx_->param_.tenant_id_, ls_partition_ids_,
-                                                    target_ls_partition_ids_,
-                                                    ctx_->param_.session_count_))) {
-        LOG_WARN("fail to init fast heap table ctx", KR(ret));
-      }
-    }
     if (OB_SUCC(ret)) {
       is_inited_ = true;
     } else {
@@ -293,11 +288,6 @@ void ObTableLoadStoreCtx::destroy()
     allocator_.free(insert_table_ctx_);
     insert_table_ctx_ = nullptr;
   }
-  if (nullptr != fast_heap_table_ctx_) {
-    fast_heap_table_ctx_->~ObDirectLoadFastHeapTableContext();
-    allocator_.free(fast_heap_table_ctx_);
-    fast_heap_table_ctx_ = nullptr;
-  }
   if (nullptr != tmp_file_mgr_) {
     tmp_file_mgr_->~ObDirectLoadTmpFileManager();
     allocator_.free(tmp_file_mgr_);

src/observer/table_load/ob_table_load_store_ctx.h

@@ -25,7 +25,6 @@ namespace oceanbase
 namespace storage
 {
 class ObDirectLoadInsertTableContext;
-class ObDirectLoadFastHeapTableContext;
 class ObDirectLoadTmpFileManager;
 }  // namespace storage
 namespace share
@@ -146,7 +145,6 @@ public:
   storage::ObDirectLoadInsertTableContext *insert_table_ctx_;
   bool is_multiple_mode_;
   bool is_fast_heap_table_;
-  storage::ObDirectLoadFastHeapTableContext *fast_heap_table_ctx_;
   storage::ObDirectLoadTmpFileManager *tmp_file_mgr_;
   ObTableLoadErrorRowHandler *error_row_handler_;
   share::schema::ObSequenceSchema sequence_schema_;

src/observer/table_load/ob_table_load_trans_store.cpp

@@ -210,13 +210,14 @@ int ObTableLoadTransStoreWriter::init_session_ctx_array()
   param.table_data_desc_ = *table_data_desc_;
   param.datum_utils_ = &(trans_ctx_->ctx_->schema_.datum_utils_);
   param.col_descs_ = &(trans_ctx_->ctx_->schema_.column_descs_);
+  param.lob_column_cnt_ = trans_ctx_->ctx_->schema_.lob_column_cnt_;
   param.cmp_funcs_ = &(trans_ctx_->ctx_->schema_.cmp_funcs_);
   param.file_mgr_ = trans_ctx_->ctx_->store_ctx_->tmp_file_mgr_;
   param.is_multiple_mode_ = trans_ctx_->ctx_->store_ctx_->is_multiple_mode_;
   param.is_fast_heap_table_ = trans_ctx_->ctx_->store_ctx_->is_fast_heap_table_;
   param.online_opt_stat_gather_ = trans_ctx_->ctx_->param_.online_opt_stat_gather_;
+  param.px_mode_ = trans_ctx_->ctx_->param_.px_mode_;
   param.insert_table_ctx_ = trans_ctx_->ctx_->store_ctx_->insert_table_ctx_;
-  param.fast_heap_table_ctx_ = trans_ctx_->ctx_->store_ctx_->fast_heap_table_ctx_;
   param.dml_row_handler_ = trans_ctx_->ctx_->store_ctx_->error_row_handler_;
   for (int64_t i = 0; OB_SUCC(ret) && i < session_count; ++i) {
     SessionContext *session_ctx = session_ctx_array_ + i;

src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.cpp

@@ -122,7 +122,7 @@ int ObAllVirtualTabletDDLKVInfo::get_next_ddl_kv(ObDDLKV *&ddl_kv)
   int ret = OB_SUCCESS;
   ObTabletHandle tablet_handle;
   while (OB_SUCC(ret)) {
-    if (ddl_kv_idx_ < 0 || ddl_kv_idx_ >= ddl_kvs_handle_.get_count()) {
+    if (ddl_kv_idx_ < 0 || ddl_kv_idx_ >= ddl_kvs_handle_.count()) {
       ObDDLKvMgrHandle ddl_kv_mgr_handle;
       if (OB_FAIL(get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) {
         if (OB_ITER_END != ret) {
@@ -130,13 +130,13 @@ int ObAllVirtualTabletDDLKVInfo::get_next_ddl_kv(ObDDLKV *&ddl_kv)
         }
       } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_kvs(false/*frozen_only*/, ddl_kvs_handle_))) {
         SERVER_LOG(WARN, "fail to get ddl kvs", K(ret));
-      } else if (ddl_kvs_handle_.get_count() > 0) {
+      } else if (ddl_kvs_handle_.count() > 0) {
         ddl_kv_idx_ = 0;
       }
     }
 
-    if (OB_SUCC(ret) && ddl_kv_idx_ >= 0 && ddl_kv_idx_ < ddl_kvs_handle_.get_count()) {
-      ddl_kv = static_cast<ObDDLKV *>(ddl_kvs_handle_.get_table(ddl_kv_idx_));
+    if (OB_SUCC(ret) && ddl_kv_idx_ >= 0 && ddl_kv_idx_ < ddl_kvs_handle_.count()) {
+      ddl_kv = ddl_kvs_handle_.at(ddl_kv_idx_).get_obj();
       if (OB_ISNULL(ddl_kv)) {
         ret = OB_ERR_UNEXPECTED;
         SERVER_LOG(WARN, "fail to get ddl kv", K(ret), K(ddl_kv_idx_));

src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h

@@ -48,7 +48,7 @@ private:
   int64_t ls_id_;
   ObSharedGuard<storage::ObLSIterator> ls_iter_guard_;
   storage::ObLSTabletIterator ls_tablet_iter_;
-  ObTablesHandleArray ddl_kvs_handle_;
+  ObArray<ObDDLKVHandle> ddl_kvs_handle_;
   common::ObTabletID curr_tablet_id_;
   int64_t ddl_kv_idx_;
   char ip_buf_[common::OB_IP_STR_BUFF];

src/rootserver/ddl_task/ob_column_redefinition_task.cpp

@@ -41,24 +41,24 @@ ObColumnRedefinitionTask::~ObColumnRedefinitionTask()
 
 int ObColumnRedefinitionTask::init(const uint64_t tenant_id, const int64_t task_id, const share::ObDDLType &ddl_type,
     const int64_t data_table_id, const int64_t dest_table_id, const int64_t schema_version, const int64_t parallelism, const int64_t consumer_group_id,
-    const int32_t sub_task_trace_id, const obrpc::ObAlterTableArg &alter_table_arg, const int64_t task_status, const int64_t snapshot_version)
+    const int32_t sub_task_trace_id, const obrpc::ObAlterTableArg &alter_table_arg,
+    const uint64_t tenant_data_version, const int64_t task_status, const int64_t snapshot_version)
 {
   int ret = OB_SUCCESS;
-  uint64_t tenant_data_format_version = 0;
   if (OB_UNLIKELY(is_inited_)) {
     ret = OB_INIT_TWICE;
     LOG_WARN("ObColumnRedefinitionTask has already been inited", K(ret));
-  } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || OB_INVALID_ID == data_table_id || OB_INVALID_ID == dest_table_id || schema_version <= 0 || task_status < ObDDLTaskStatus::PREPARE
+  } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || OB_INVALID_ID == data_table_id || OB_INVALID_ID == dest_table_id || schema_version <= 0
+      || tenant_data_version <= 0 || task_status < ObDDLTaskStatus::PREPARE
       || task_status > ObDDLTaskStatus::SUCCESS || snapshot_version < 0 || (snapshot_version > 0 && task_status < ObDDLTaskStatus::WAIT_TRANS_END))) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(task_id), K(data_table_id), K(dest_table_id), K(schema_version), K(task_status), K(snapshot_version));
+    LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(task_id), K(data_table_id), K(dest_table_id), K(schema_version),
+      K(tenant_data_version), K(task_status), K(snapshot_version));
     LOG_WARN("fail to init task table operator", K(ret));
   } else if (OB_FAIL(deep_copy_table_arg(allocator_, alter_table_arg, alter_table_arg_))) {
     LOG_WARN("deep copy alter table arg failed", K(ret));
   } else if (OB_FAIL(set_ddl_stmt_str(alter_table_arg_.ddl_stmt_str_))) {
     LOG_WARN("set ddl stmt str failed", K(ret));
-  } else if (OB_FAIL(ObShareUtil::fetch_current_data_version(*GCTX.sql_proxy_, tenant_id, tenant_data_format_version))) {
-    LOG_WARN("get min data version failed", K(ret), K(tenant_id));
   } else {
     set_gmt_create(ObTimeUtility::current_time());
     task_type_ = ddl_type;
@@ -83,7 +83,7 @@ int ObColumnRedefinitionTask::init(const uint64_t tenant_id, const int64_t task_
       alter_table_arg_.alter_table_schema_.set_tenant_id(tenant_id_);
       alter_table_arg_.alter_table_schema_.set_schema_version(schema_version_);
       alter_table_arg_.exec_tenant_id_ = dst_tenant_id_;
-      data_format_version_ = tenant_data_format_version;
+      data_format_version_ = tenant_data_version;
       is_inited_ = true;
       ddl_tracing_.open();
     }
@@ -284,6 +284,7 @@ int ObColumnRedefinitionTask::copy_table_indexes()
                                          &create_index_arg,
                                          task_id_);
               param.sub_task_trace_id_ = sub_task_trace_id_;
+              param.tenant_data_version_ = data_format_version_;
               if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param,
                                                                                        *GCTX.sql_proxy_,
                                                                                        task_record))) {

src/rootserver/ddl_task/ob_column_redefinition_task.h

@@ -39,6 +39,7 @@ public:
       const int64_t consumer_group_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg &alter_table_arg,
+      const uint64_t tenant_data_version,
       const int64_t task_status = share::ObDDLTaskStatus::PREPARE,
       const int64_t snapshot_version = 0);
   int init(const ObDDLTaskRecord &task_record);

src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp

@@ -51,13 +51,14 @@ ObDDLRedefinitionSSTableBuildTask::ObDDLRedefinitionSSTableBuildTask(
     const bool is_mview_complete_refresh,
     const int64_t mview_table_id,
     ObRootService *root_service,
-    const common::ObAddr &inner_sql_exec_addr)
+    const common::ObAddr &inner_sql_exec_addr,
+    const int64_t data_format_version)
   : is_inited_(false), tenant_id_(tenant_id), task_id_(task_id), data_table_id_(data_table_id),
     dest_table_id_(dest_table_id), schema_version_(schema_version), snapshot_version_(snapshot_version),
     execution_id_(execution_id), consumer_group_id_(consumer_group_id), sql_mode_(sql_mode), trace_id_(trace_id),
     parallelism_(parallelism), use_heap_table_ddl_plan_(use_heap_table_ddl_plan),
     is_mview_complete_refresh_(is_mview_complete_refresh), mview_table_id_(mview_table_id),
-    root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr)
+    root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr), data_format_version_(0)
 {
   set_retry_times(0); // do not retry
 }
@@ -196,10 +197,41 @@ int ObDDLRedefinitionSSTableBuildTask::process()
           LOG_WARN("ddl sim failure", K(ret), K(tenant_id_), K(task_id_));
         } else if (OB_FAIL(user_sql_proxy->write(tenant_id_, sql_string.ptr(), affected_rows,
                 oracle_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE, &session_param, sql_exec_addr))) {
-          LOG_WARN("fail to execute build replica sql", K(ret), K(tenant_id_));
-        } else if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(tenant_id_, dest_table_id_, execution_id_, task_id_))) {
-          LOG_WARN("fail to check sstable checksum_report_finish",
-            K(ret), K(tenant_id_), K(dest_table_id_), K(execution_id_), K(task_id_));
+          if (ret == OB_SERVER_OUTOF_DISK_SPACE &&
+              data_format_version_ >= DATA_VERSION_4_3_0_0) {
+            // if version >= 4.3.0, would retry with compression.
+            int tmp_ret = OB_SUCCESS;
+            sql_string.reuse();
+            SortCompactLevel compress_level = SORT_COMPRESSION_LEVEL;
+            if (OB_SUCCESS != (tmp_ret = ObDDLUtil::generate_build_replica_sql(tenant_id_, data_table_id_,
+                                                            dest_table_id_,
+                                                            data_table_schema->get_schema_version(),
+                                                            snapshot_version_,
+                                                            execution_id_,
+                                                            task_id_,
+                                                            parallelism_,
+                                                            use_heap_table_ddl_plan_,
+                                                            true,
+                                                            &col_name_map_,
+                                                            sql_string,
+                                                            compress_level))) {
+              LOG_WARN("fail to generate build replica sql", K(tmp_ret));
+            } else if (OB_SUCCESS != (tmp_ret = user_sql_proxy->write(tenant_id_, sql_string.ptr(), affected_rows,
+                oracle_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE,
+                &session_param, sql_exec_addr))) {
+              LOG_WARN("fail to execute build replica sql", K(tmp_ret), K(tenant_id_));
+            } else {
+              ret = OB_SUCCESS;
+            }
+          } else {
+            LOG_WARN("fail to execute build replica sql", K(ret), K(tenant_id_));
+          }
+        }
+        if (OB_SUCC(ret)) {
+          if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(tenant_id_, dest_table_id_, execution_id_, task_id_))) {
+            LOG_WARN("fail to check sstable checksum_report_finish",
+              K(ret), K(tenant_id_), K(dest_table_id_), K(execution_id_), K(task_id_));
+          }
         }
       }
     }
@@ -251,7 +283,8 @@ ObAsyncTask *ObDDLRedefinitionSSTableBuildTask::deep_copy(char *buf, const int64
         is_mview_complete_refresh_,
         mview_table_id_,
         root_service_,
-        inner_sql_exec_addr_);
+        inner_sql_exec_addr_,
+        data_format_version_);
     if (OB_FAIL(new_task->tz_info_wrap_.deep_copy(tz_info_wrap_))) {
       LOG_WARN("failed to copy tz info wrap", K(ret));
     } else if (OB_FAIL(new_task->col_name_map_.assign(col_name_map_))) {

src/rootserver/ddl_task/ob_ddl_redefinition_task.h

@@ -41,7 +41,8 @@ public:
       const bool is_mview_complete_refresh,
       const int64_t mview_table_id,
       ObRootService *root_service,
-      const common::ObAddr &inner_sql_exec_addr);
+      const common::ObAddr &inner_sql_exec_addr,
+      const int64_t data_format_version = 0);
   int init(
       const ObTableSchema &orig_table_schema,
       const AlterTableSchema &alter_table_schema,
@@ -74,6 +75,7 @@ private:
   common::ObArray<share::schema::ObBasedSchemaObjectInfo> based_schema_object_infos_;
   ObRootService *root_service_;
   common::ObAddr inner_sql_exec_addr_;
+  int64_t data_format_version_;
 };
 
 class ObSyncTabletAutoincSeqCtx final

src/rootserver/ddl_task/ob_ddl_scheduler.cpp

@@ -936,6 +936,23 @@ void ObDDLScheduler::run1()
   }
 }
 
+int ObDDLScheduler::check_conflict_with_upgrade(
+    const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!is_inited_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ObDDLScheduler has not been inited", K(ret));
+  } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid arg", K(ret), K(tenant_id));
+  } else if (GCONF.in_upgrade_mode()) {
+    ret = OB_NOT_SUPPORTED;
+    LOG_WARN("Ddl task is disallowed to create when upgrading", K(ret));
+  }
+  return ret;
+}
+
 int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
                                     ObISQLClient &proxy,
                                     ObDDLTaskRecord &task_record)
@@ -947,17 +964,12 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
   const obrpc::ObDropIndexArg *drop_index_arg = nullptr;
   const obrpc::ObMViewCompleteRefreshArg *mview_complete_refresh_arg = nullptr;
   ObRootService *root_service = GCTX.root_service_;
-  uint64_t tenant_id = param.tenant_id_;
-  uint64_t compat_version = 0;
   LOG_INFO("create ddl task", K(param));
-  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
-    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
-  } else if (compat_version < DATA_VERSION_4_1_0_0 && GCONF.in_upgrade_mode()) {
-    ret = OB_NOT_SUPPORTED;
-    LOG_WARN("4.0 is being upgrade to 4.1, create_ddl_task not supported", K(ret));
-  } else if (OB_UNLIKELY(!is_inited_)) {
+  if (OB_UNLIKELY(!is_inited_)) {
     ret = OB_NOT_INIT;
     LOG_WARN("ObDDLScheduler has not been inited", K(ret));
+  } else if (OB_FAIL(check_conflict_with_upgrade(param.tenant_id_))) {
+    LOG_WARN("conflict with upgrade", K(ret), K(param));
   } else if (OB_ISNULL(root_service)) {
     ret = OB_ERR_SYS;
     LOG_WARN("error sys, root service must not be nullptr", K(ret));
@@ -978,6 +990,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
                                             param.consumer_group_id_,
                                             param.sub_task_trace_id_,
                                             create_index_arg,
+                                            param.tenant_data_version_,
                                             *param.allocator_,
                                             task_record))) {
           LOG_WARN("fail to create build index task", K(ret));
@@ -1007,6 +1020,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
       case DDL_TABLE_REDEFINITION:
       case DDL_DIRECT_LOAD:
       case DDL_DIRECT_LOAD_INSERT:
+      case DDL_ALTER_COLUMN_GROUP:
       case DDL_MVIEW_COMPLETE_REFRESH:
         if (OB_FAIL(create_table_redefinition_task(proxy,
                                                    param.type_,
@@ -1017,6 +1031,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
                                                    param.task_id_,
                                                    param.sub_task_trace_id_,
                                                    static_cast<const obrpc::ObAlterTableArg *>(param.ddl_arg_),
+                                                   param.tenant_data_version_,
                                                    *param.allocator_,
                                                    task_record))) {
           LOG_WARN("fail to create table redefinition task", K(ret));
@@ -1045,6 +1060,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
                                                    param.task_id_,
                                                    param.sub_task_trace_id_,
                                                    static_cast<const obrpc::ObAlterTableArg *>(param.ddl_arg_),
+                                                   param.tenant_data_version_,
                                                    *param.allocator_,
                                                    task_record))) {
           LOG_WARN("fail to create recover restore table task", K(ret));
@@ -1061,6 +1077,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
                                                  param.task_id_,
                                                  param.sub_task_trace_id_,
                                                  alter_table_arg,
+                                                 param.tenant_data_version_,
                                                  *param.allocator_,
                                                  task_record))) {
           LOG_WARN("fail to create table redefinition task", K(ret));
@@ -1095,6 +1112,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
                                                     param.task_id_,
                                                     param.sub_task_trace_id_,
                                                     static_cast<const obrpc::ObAlterTableArg *>(param.ddl_arg_),
+                                                    param.tenant_data_version_,
                                                     *param.allocator_,
                                                     task_record))) {
           LOG_WARN("fail to create column redefinition task", K(ret));
@@ -1486,6 +1504,7 @@ int ObDDLScheduler::create_build_index_task(
     const int64_t consumer_group_id,
     const int32_t sub_task_trace_id,
     const obrpc::ObCreateIndexArg *create_index_arg,
+    const uint64_t tenant_data_version,
     ObIAllocator &allocator,
     ObDDLTaskRecord &task_record)
 {
@@ -1495,9 +1514,10 @@ int ObDDLScheduler::create_build_index_task(
     if (OB_UNLIKELY(!is_inited_)) {
       ret = OB_NOT_INIT;
       LOG_WARN("not init", K(ret));
-    } else if (OB_ISNULL(create_index_arg) || OB_ISNULL(data_table_schema) || OB_ISNULL(index_schema)) {
+    } else if (OB_ISNULL(create_index_arg) || OB_ISNULL(data_table_schema) || OB_ISNULL(index_schema)
+        || OB_UNLIKELY(tenant_data_version <= 0)) {
       ret = OB_INVALID_ARGUMENT;
-      LOG_WARN("invalid argument", K(ret), K(create_index_arg), K(data_table_schema), K(index_schema));
+      LOG_WARN("invalid argument", K(ret), K(create_index_arg), K(data_table_schema), K(index_schema), K(tenant_data_version));
     } else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), data_table_schema->get_tenant_id(), task_id))) {
       LOG_WARN("fetch new task id failed", K(ret));
     } else if (OB_FAIL(index_task.init(data_table_schema->get_tenant_id(),
@@ -1510,7 +1530,8 @@ int ObDDLScheduler::create_build_index_task(
                                       consumer_group_id,
                                       sub_task_trace_id,
                                       *create_index_arg,
-                                      parent_task_id))) {
+                                      parent_task_id,
+                                      tenant_data_version))) {
       LOG_WARN("init global index task failed", K(ret), K(data_table_schema), K(index_schema));
     } else if (OB_FAIL(index_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
       LOG_WARN("set trace id failed", K(ret));
@@ -1618,29 +1639,30 @@ int ObDDLScheduler::create_table_redefinition_task(
     const int64_t task_id,
     const int32_t sub_task_trace_id,
     const obrpc::ObAlterTableArg *alter_table_arg,
+    const uint64_t tenant_data_version,
     ObIAllocator &allocator,
     ObDDLTaskRecord &task_record)
 {
   int ret = OB_SUCCESS;
+  int64_t target_cg_cnt = 0;
   SMART_VAR(ObTableRedefinitionTask, redefinition_task) {
     if (OB_UNLIKELY(!is_inited_)) {
       ret = OB_NOT_INIT;
       LOG_WARN("ObDDLScheduler has not been inited", K(ret));
-    } else if (OB_UNLIKELY(0 == task_id) || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
+    } else if (OB_UNLIKELY(0 == task_id || tenant_data_version <= 0) || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
       ret = OB_INVALID_ARGUMENT;
-      LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema));
-    } else if (OB_FAIL(redefinition_task.init(src_schema->get_tenant_id(),
-                                              dest_schema->get_tenant_id(),
+      LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema),  K(tenant_data_version));
+    } else if (OB_FAIL(dest_schema->get_store_column_group_count(target_cg_cnt))) {
+      LOG_WARN("fail to get target_cg_cnt", K(ret), K(dest_schema));
+    } else if (OB_FAIL(redefinition_task.init(src_schema,
+                                              dest_schema,
                                               task_id,
                                               type,
-                                              src_schema->get_table_id(),
-                                              dest_schema->get_table_id(),
-                                              dest_schema->get_schema_version(),
-                                              dest_schema->get_schema_version(),
                                               parallelism,
                                               consumer_group_id,
                                               sub_task_trace_id,
-                                              *alter_table_arg))) {
+                                              *alter_table_arg,
+                                              tenant_data_version))) {
       LOG_WARN("fail to init redefinition task", K(ret));
     } else if (OB_FAIL(redefinition_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
       LOG_WARN("set trace id failed", K(ret));
@@ -1662,27 +1684,30 @@ int ObDDLScheduler::create_drop_primary_key_task(
     const int64_t task_id,
     const int32_t sub_task_trace_id,
     const obrpc::ObAlterTableArg *alter_table_arg,
+    const uint64_t tenant_data_version,
     ObIAllocator &allocator,
     ObDDLTaskRecord &task_record)
 {
   int ret = OB_SUCCESS;
+  int64_t target_cg_cnt = 0;
   SMART_VAR(ObDropPrimaryKeyTask, drop_pk_task) {
   if (OB_UNLIKELY(!is_inited_)) {
     ret = OB_NOT_INIT;
     LOG_WARN("ObDDLScheduler has not been inited", K(ret));
-  } else if (OB_UNLIKELY(0 == task_id) || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
+  } else if (OB_UNLIKELY(0 == task_id || tenant_data_version <= 0) || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema));
-  } else if (OB_FAIL(drop_pk_task.init(src_schema->get_tenant_id(),
+    LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema), K(tenant_data_version));
+  } else if (OB_FAIL(dest_schema->get_store_column_group_count(target_cg_cnt))) {
+    LOG_WARN("fail to get target_store_cg_cnt", K(ret), KPC(dest_schema));
+  } else if (OB_FAIL(drop_pk_task.init(src_schema,
+                                       dest_schema,
                                        task_id,
                                        type,
-                                       src_schema->get_table_id(),
-                                       dest_schema->get_table_id(),
-                                       dest_schema->get_schema_version(),
                                        parallelism,
                                        consumer_group_id,
                                        sub_task_trace_id,
-                                       *alter_table_arg))) {
+                                       *alter_table_arg,
+                                       tenant_data_version))) {
     LOG_WARN("fail to init redefinition task", K(ret));
   } else if (OB_FAIL(drop_pk_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
     LOG_WARN("set trace id failed", K(ret));
@@ -1704,17 +1729,20 @@ int ObDDLScheduler::create_column_redefinition_task(
     const int64_t task_id,
     const int32_t sub_task_trace_id,
     const obrpc::ObAlterTableArg *alter_table_arg,
+    const uint64_t tenant_data_version,
     ObIAllocator &allocator,
     ObDDLTaskRecord &task_record)
 {
   int ret = OB_SUCCESS;
+  int64_t target_cg_cnt = 0;
   SMART_VAR(ObColumnRedefinitionTask, redefinition_task) {
   if (OB_UNLIKELY(!is_inited_)) {
     ret = OB_NOT_INIT;
     LOG_WARN("ObDDLScheduler has not been inited", K(ret));
-  } else if (OB_UNLIKELY(0 == task_id) || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
+  } else if (OB_UNLIKELY(0 == task_id || tenant_data_version <= 0)
+    || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema));
+    LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema), K(tenant_data_version));
   } else if (OB_FAIL(redefinition_task.init(src_schema->get_tenant_id(),
                                             task_id,
                                             type,
@@ -1724,7 +1752,8 @@ int ObDDLScheduler::create_column_redefinition_task(
                                             parallelism,
                                             consumer_group_id,
                                             sub_task_trace_id,
-                                            *alter_table_arg))) {
+                                            *alter_table_arg,
+                                            tenant_data_version))) {
     LOG_WARN("fail to init redefinition task", K(ret));
   } else if (OB_FAIL(redefinition_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
     LOG_WARN("set trace id failed", K(ret));
@@ -1816,29 +1845,31 @@ int ObDDLScheduler::create_recover_restore_table_task(
     const int64_t task_id,
     const int32_t sub_task_trace_id,
     const obrpc::ObAlterTableArg *alter_table_arg,
+    const uint64_t tenant_data_version,
     ObIAllocator &allocator,
     ObDDLTaskRecord &task_record)
 {
   int ret = OB_SUCCESS;
+  int64_t target_cg_cnt = 0;
   SMART_VAR(ObRecoverRestoreTableTask, redefinition_task) {
     if (OB_UNLIKELY(!is_inited_)) {
       ret = OB_NOT_INIT;
       LOG_WARN("ObDDLScheduler has not been inited", K(ret));
-    } else if (OB_UNLIKELY(0 == task_id) || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
+    } else if (OB_UNLIKELY(0 == task_id || tenant_data_version <= 0)
+        || OB_ISNULL(alter_table_arg) || OB_ISNULL(src_schema) || OB_ISNULL(dest_schema)) {
       ret = OB_INVALID_ARGUMENT;
-      LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema));
-    } else if (OB_FAIL(redefinition_task.init(src_schema->get_tenant_id(),
-                                              dest_schema->get_tenant_id(),
+      LOG_WARN("invalid arguments", K(ret), K(task_id), KP(alter_table_arg), KP(src_schema), KP(dest_schema), K(tenant_data_version));
+    } else if (OB_FAIL(dest_schema->get_store_column_group_count(target_cg_cnt))) {
+      LOG_WARN("fail to get store cg cnt", K(ret), KPC(dest_schema));
+    } else if (OB_FAIL(redefinition_task.init(src_schema,
+                                              dest_schema,
                                               task_id,
                                               type,
-                                              src_schema->get_table_id(),
-                                              dest_schema->get_table_id(),
-                                              src_schema->get_schema_version(),
-                                              dest_schema->get_schema_version(),
                                               parallelism,
                                               consumer_group_id,
                                               sub_task_trace_id,
-                                              *alter_table_arg))) {
+                                              *alter_table_arg,
+                                              tenant_data_version))) {
       LOG_WARN("fail to init redefinition task", K(ret));
     } else if (OB_FAIL(redefinition_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
       LOG_WARN("set trace id failed", K(ret));
@@ -2033,6 +2064,7 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record)
       case DDL_TABLE_REDEFINITION:
       case DDL_DIRECT_LOAD:
       case DDL_DIRECT_LOAD_INSERT:
+      case DDL_ALTER_COLUMN_GROUP:
       case DDL_MVIEW_COMPLETE_REFRESH:
         ret = schedule_table_redefinition_task(record);
         break;
@@ -2602,6 +2634,7 @@ int ObDDLScheduler::on_sstable_complement_job_reply(
           case ObDDLType::DDL_MODIFY_COLUMN:
           case ObDDLType::DDL_CONVERT_TO_CHARACTER:
           case ObDDLType::DDL_TABLE_REDEFINITION:
+          case ObDDLType::DDL_ALTER_COLUMN_GROUP:
           case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH:
             if (OB_FAIL(static_cast<ObTableRedefinitionTask *>(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
               LOG_WARN("update complete sstable job status", K(ret));
@@ -2734,6 +2767,7 @@ int ObDDLScheduler::notify_update_autoinc_end(const ObDDLTaskKey &task_key,
           case ObDDLType::DDL_TABLE_REDEFINITION:
           case ObDDLType::DDL_DIRECT_LOAD:
           case ObDDLType::DDL_DIRECT_LOAD_INSERT:
+          case ObDDLType::DDL_ALTER_COLUMN_GROUP:
           case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH:
             if (OB_FAIL(static_cast<ObTableRedefinitionTask *>(&task)->notify_update_autoinc_finish(autoinc_val, ret_code))) {
               LOG_WARN("update complete sstable job status", K(ret));

src/rootserver/ddl_task/ob_ddl_scheduler.h

@@ -362,6 +362,7 @@ private:
       const int64_t consumer_group_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObCreateIndexArg *create_index_arg,
+      const uint64_t tenant_data_version,
       ObIAllocator &allocator,
       ObDDLTaskRecord &task_record);
   int create_constraint_task(
@@ -395,6 +396,7 @@ private:
       const int64_t task_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg *alter_table_arg,
+      const uint64_t tenant_data_version,
       ObIAllocator &allocator,
       ObDDLTaskRecord &task_record);
 
@@ -408,6 +410,7 @@ private:
       const int64_t task_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg *alter_table_arg,
+      const uint64_t tenant_data_version,
       ObIAllocator &allocator,
       ObDDLTaskRecord &task_record);
 
@@ -421,6 +424,7 @@ private:
       const int64_t task_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg *alter_table_arg,
+      const uint64_t tenant_data_version,
       ObIAllocator &allocator,
       ObDDLTaskRecord &task_record);
 
@@ -470,6 +474,7 @@ private:
       const int64_t task_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg *alter_table_arg,
+      const uint64_t tenant_data_version,
       ObIAllocator &allocator,
       ObDDLTaskRecord &task_record);
 
@@ -490,6 +495,8 @@ private:
   int remove_task_from_longops_mgr(ObDDLTask *ddl_task);
   int remove_ddl_task(ObDDLTask *ddl_task);
   void add_event_info(const ObDDLTaskRecord &ddl_record, const ObString &ddl_event_stmt);
+  int check_conflict_with_upgrade(
+      const uint64_t tenant_id);
 
 private:
   static const int64_t TOTAL_LIMIT = 1024L * 1024L * 1024L;

src/rootserver/ddl_task/ob_ddl_task.cpp

@@ -130,7 +130,7 @@ int ObDDLTaskID::assign(const ObDDLTaskID &other)
 
 ObDDLTaskSerializeField::ObDDLTaskSerializeField(const int64_t task_version,
                                                  const int64_t parallelism,
-                                                 const int64_t data_format_version,
+                                                 const uint64_t data_format_version,
                                                  const int64_t consumer_group_id,
                                                  const bool is_abort,
                                                  const int32_t sub_task_trace_id)
@@ -162,8 +162,9 @@ OB_SERIALIZE_MEMBER(ObDDLTaskSerializeField,
                     sub_task_trace_id_);
 
 ObCreateDDLTaskParam::ObCreateDDLTaskParam()
-  : sub_task_trace_id_(0), tenant_id_(OB_INVALID_ID), object_id_(OB_INVALID_ID), schema_version_(0), parallelism_(0), consumer_group_id_(0), parent_task_id_(0), task_id_(0),
-    type_(DDL_INVALID), src_table_schema_(nullptr), dest_table_schema_(nullptr), ddl_arg_(nullptr), allocator_(nullptr)
+  : sub_task_trace_id_(0), tenant_id_(OB_INVALID_ID), object_id_(OB_INVALID_ID), schema_version_(0), parallelism_(0),
+    consumer_group_id_(0), parent_task_id_(0), task_id_(0), type_(DDL_INVALID), src_table_schema_(nullptr),
+    dest_table_schema_(nullptr), ddl_arg_(nullptr), allocator_(nullptr), tenant_data_version_(0)
 {
 }
 
@@ -277,6 +278,7 @@ trace::ObSpanCtx* ObDDLTracing::begin_task_span()
       case DDL_ALTER_PARTITION_BY:
       case DDL_CONVERT_TO_CHARACTER:
       case DDL_TABLE_REDEFINITION:
+      case DDL_ALTER_COLUMN_GROUP:
         span = FLT_BEGIN_SPAN(ddl_table_redefinition);
         break;
       case DDL_DROP_PRIMARY_KEY:
@@ -359,6 +361,7 @@ trace::ObSpanCtx* ObDDLTracing::restore_task_span()
       case DDL_ALTER_PARTITION_BY:
       case DDL_CONVERT_TO_CHARACTER:
       case DDL_TABLE_REDEFINITION:
+      case DDL_ALTER_COLUMN_GROUP:
         span = FLT_RESTORE_DDL_SPAN(ddl_table_redefinition, task_span_id_, task_start_ts_);
         break;
       case DDL_DROP_PRIMARY_KEY:
@@ -784,6 +787,9 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st
     case DDL_DROP_INDEX:
       ddl_type_str = "drop index";
       break;
+    case DDL_ALTER_COLUMN_GROUP:
+      ddl_type_str = "alter column group";
+      break;
     case DDL_MVIEW_COMPLETE_REFRESH:
       ddl_type_str = "mview complete refresh";
       break;
@@ -1627,10 +1633,12 @@ int ObDDLTask::gather_inserted_rows(
     const uint64_t tenant_id,
     const int64_t task_id,
     ObMySQLProxy &sql_proxy,
-    int64_t &row_inserted)
+    int64_t &row_inserted_cg,
+    int64_t &row_inserted_file)
 {
   int ret = OB_SUCCESS;
-  row_inserted = 0;
+  row_inserted_cg = 0;
+  row_inserted_file = 0;
   char trace_id_str[OB_MAX_TRACE_ID_BUFFER_SIZE] = "";
   trace_id_.to_string(trace_id_str, OB_MAX_TRACE_ID_BUFFER_SIZE);
   if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || task_id <= 0)) {
@@ -1641,7 +1649,7 @@ int ObDDLTask::gather_inserted_rows(
     sqlclient::ObMySQLResult *insert_result = NULL;
     SMART_VAR(ObMySQLProxy::MySQLResult, insert_res) {
       if (OB_FAIL(insert_sql.assign_fmt(
-          "SELECT OTHERSTAT_1_VALUE AS ROW_INSERTED FROM %s WHERE TENANT_ID=%lu "
+          "SELECT OTHERSTAT_1_VALUE AS CG_ROW_INSERTED, OTHERSTAT_2_VALUE AS SSTABLE_ROW_INSERTED FROM %s WHERE TENANT_ID=%lu "
           "AND TRACE_ID='%s' AND PLAN_OPERATION='PHY_PX_MULTI_PART_SSTABLE_INSERT' AND OTHERSTAT_5_VALUE='%ld'",
           OB_ALL_VIRTUAL_SQL_PLAN_MONITOR_TNAME, tenant_id, trace_id_str, task_id))) {
         LOG_WARN("failed to assign sql", K(ret));
@@ -1662,9 +1670,13 @@ int ObDDLTask::gather_inserted_rows(
             LOG_WARN("failed to get next row", K(ret));
           }
         } else {
-          int64_t row_inserted_tmp = 0;
-          EXTRACT_INT_FIELD_MYSQL(*insert_result, "ROW_INSERTED", row_inserted_tmp, int64_t);
-          row_inserted += row_inserted_tmp;
+          int64_t row_inserted_cg_tmp = 0;
+          EXTRACT_INT_FIELD_MYSQL(*insert_result, "CG_ROW_INSERTED", row_inserted_cg_tmp, int64_t);
+          row_inserted_cg += row_inserted_cg_tmp;
+
+          int64_t row_inserted_file_tmp = 0;
+          EXTRACT_INT_FIELD_MYSQL(*insert_result, "SSTABLE_ROW_INSERTED", row_inserted_file_tmp, int64_t);
+          row_inserted_file += row_inserted_file_tmp;
         }
       }
     }
@@ -1677,12 +1689,14 @@ int ObDDLTask::gather_redefinition_stats(const uint64_t tenant_id,
                                          ObMySQLProxy &sql_proxy,
                                          int64_t &row_scanned,
                                          int64_t &row_sorted,
-                                         int64_t &row_inserted)
+                                         int64_t &row_inserted_cg,
+                                         int64_t &row_inserted_file)
 {
   int ret = OB_SUCCESS;
   row_scanned = 0;
   row_sorted = 0;
-  row_inserted = 0;
+  row_inserted_cg = 0;
+  row_inserted_file = 0;
   if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || task_id <= 0)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id));
@@ -1690,7 +1704,7 @@ int ObDDLTask::gather_redefinition_stats(const uint64_t tenant_id,
     LOG_WARN("gather scanned rows failed", K(ret));
   } else if (OB_FAIL(gather_sorted_rows(tenant_id, task_id, sql_proxy, row_sorted))) {
     LOG_WARN("gather sorted rows failed", K(ret));
-  } else if (OB_FAIL(gather_inserted_rows(tenant_id, task_id, sql_proxy, row_inserted))) {
+  } else if (OB_FAIL(gather_inserted_rows(tenant_id, task_id, sql_proxy, row_inserted_cg, row_inserted_file))) {
     LOG_WARN("gather inserted rows failed", K(ret));
   }
   return ret;
@@ -2957,33 +2971,33 @@ int ObDDLTaskRecordOperator::check_has_conflict_ddl(
             LOG_WARN("failed to fill task record", K(ret));
           } else if (task_record.task_id_ != task_id) {
             switch (ddl_type) {
-            case ObDDLType::DDL_DROP_TABLE: {
-              if (task_record.ddl_type_ == ObDDLType::DDL_DROP_INDEX && task_record.target_object_id_ != task_record.object_id_) {
-                LOG_WARN("conflict with ddl", K(task_record));
-                has_conflict_ddl = true;
+              case ObDDLType::DDL_DROP_TABLE: {
+                if (task_record.ddl_type_ == ObDDLType::DDL_DROP_INDEX && task_record.target_object_id_ != task_record.object_id_) {
+                  LOG_WARN("conflict with ddl", K(task_record));
+                  has_conflict_ddl = true;
+                }
+                break;
               }
-              break;
-            }
-            case ObDDLType::DDL_DOUBLE_TABLE_OFFLINE:
-            case ObDDLType::DDL_MODIFY_COLUMN:
-            case ObDDLType::DDL_ADD_PRIMARY_KEY:
-            case ObDDLType::DDL_DROP_PRIMARY_KEY:
-            case ObDDLType::DDL_ALTER_PRIMARY_KEY:
-            case ObDDLType::DDL_ALTER_PARTITION_BY:
-            case ObDDLType::DDL_DROP_COLUMN:
-            case ObDDLType::DDL_CONVERT_TO_CHARACTER:
-            case ObDDLType::DDL_ADD_COLUMN_OFFLINE:
-            case ObDDLType::DDL_COLUMN_REDEFINITION:
-            case ObDDLType::DDL_TABLE_REDEFINITION:
-            case ObDDLType::DDL_DIRECT_LOAD:
-            case ObDDLType::DDL_DIRECT_LOAD_INSERT:
-            case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH: {
-              has_conflict_ddl = true;
-              break;
-            }
-            default: {
-              // do nothing
-            }
+              case ObDDLType::DDL_DOUBLE_TABLE_OFFLINE:
+              case ObDDLType::DDL_MODIFY_COLUMN:
+              case ObDDLType::DDL_ADD_PRIMARY_KEY:
+              case ObDDLType::DDL_DROP_PRIMARY_KEY:
+              case ObDDLType::DDL_ALTER_PRIMARY_KEY:
+              case ObDDLType::DDL_ALTER_PARTITION_BY:
+              case ObDDLType::DDL_DROP_COLUMN:
+              case ObDDLType::DDL_CONVERT_TO_CHARACTER:
+              case ObDDLType::DDL_ADD_COLUMN_OFFLINE:
+              case ObDDLType::DDL_COLUMN_REDEFINITION:
+              case ObDDLType::DDL_TABLE_REDEFINITION:
+              case ObDDLType::DDL_DIRECT_LOAD:
+              case ObDDLType::DDL_DIRECT_LOAD_INSERT:
+              case ObDDLType::DDL_ALTER_COLUMN_GROUP:
+              case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH:
+                has_conflict_ddl = true;
+                break;
+              default:
+                // do nothing
+                break;
             }
           }
         }

src/rootserver/ddl_task/ob_ddl_task.h

@@ -117,7 +117,7 @@ public:
   ObDDLTaskSerializeField() : task_version_(0), parallelism_(0), data_format_version_(0), consumer_group_id_(0), is_abort_(false), sub_task_trace_id_(0) {}
   ObDDLTaskSerializeField(const int64_t task_version,
                           const int64_t parallelism,
-                          const int64_t data_format_version,
+                          const uint64_t data_format_version,
                           const int64_t consumer_group_id,
                           const bool is_abort,
                           const int32_t sub_task_trace_id);
@@ -126,7 +126,7 @@ public:
 public:
   int64_t task_version_;
   int64_t parallelism_;
-  int64_t data_format_version_;
+  uint64_t data_format_version_;
   int64_t consumer_group_id_;
   bool is_abort_;
   int32_t sub_task_trace_id_;
@@ -151,8 +151,8 @@ public:
   ~ObCreateDDLTaskParam() = default;
   bool is_valid() const { return OB_INVALID_ID != tenant_id_ && type_ > share::DDL_INVALID
                                  && type_ < share::DDL_MAX && nullptr != allocator_; }
-  TO_STRING_KV(K_(sub_task_trace_id), K_(tenant_id), K_(object_id), K_(schema_version), K_(parallelism), K_(consumer_group_id), K_(parent_task_id), K_(task_id),
-               K_(type), KPC_(src_table_schema), KPC_(dest_table_schema), KPC_(ddl_arg));
+  TO_STRING_KV(K_(tenant_id), K_(object_id), K_(schema_version), K_(parallelism), K_(consumer_group_id), K_(parent_task_id), K_(task_id),
+               K_(type), KPC_(src_table_schema), KPC_(dest_table_schema), KPC_(ddl_arg), K_(tenant_data_version), K_(sub_task_trace_id));
 public:
   int32_t sub_task_trace_id_;
   uint64_t tenant_id_;
@@ -167,6 +167,7 @@ public:
   const ObTableSchema *dest_table_schema_;
   const obrpc::ObDDLArg *ddl_arg_;
   common::ObIAllocator *allocator_;
+  uint64_t tenant_data_version_;
 };
 
 class ObDDLTaskRecordOperator final
@@ -516,7 +517,7 @@ public:
                                  obrpc::ObDDLArg &dest_arg);
   void set_longops_stat(share::ObDDLLongopsStat *longops_stat) { longops_stat_ = longops_stat; }
   share::ObDDLLongopsStat *get_longops_stat() const { return longops_stat_; }
-  int64_t get_data_format_version() const { return data_format_version_; }
+  uint64_t get_data_format_version() const { return data_format_version_; }
   static int fetch_new_task_id(ObMySQLProxy &sql_proxy, const uint64_t tenant_id, int64_t &new_task_id);
   virtual int serialize_params_to_message(char *buf, const int64_t buf_size, int64_t &pos) const;
   virtual int deserlize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos);
@@ -575,7 +576,8 @@ protected:
                                 ObMySQLProxy &sql_proxy,
                                 int64_t &row_scanned,
                                 int64_t &row_sorted,
-                                int64_t &row_inserted);
+                                int64_t &row_inserted_cg,
+                                int64_t &row_inserted_file);
   int gather_scanned_rows(
       const uint64_t tenant_id,
       const int64_t task_id,
@@ -590,7 +592,8 @@ protected:
       const uint64_t tenant_id,
       const int64_t task_id,
       ObMySQLProxy &sql_proxy,
-      int64_t &row_inserted);
+      int64_t &row_inserted_cg,
+      int64_t &row_inserted_file);
   int copy_longops_stat(share::ObLongopsValue &value);
   virtual bool is_error_need_retry(const int ret_code)
   {
@@ -636,7 +639,7 @@ protected:
   int64_t execution_id_; // guarded by lock_
   common::ObAddr sql_exec_addr_;
   int64_t start_time_;
-  int64_t data_format_version_;
+  uint64_t data_format_version_;
   int64_t consumer_group_id_;
 };
 

src/rootserver/ddl_task/ob_drop_index_task.cpp

@@ -402,6 +402,13 @@ int ObDropIndexTask::check_switch_succ()
     LOG_WARN("error sys", K(ret));
   } else if (OB_FAIL(refresh_schema_version())) {
     LOG_WARN("refresh schema version failed", K(ret));
+  } else if (OB_FAIL(ObDDLUtil::check_tenant_status_normal(&root_service_->get_sql_proxy(), tenant_id_))) {
+    if (OB_TENANT_HAS_BEEN_DROPPED == ret || OB_STANDBY_READ_ONLY == ret) {
+      need_retry_ = false;
+      LOG_INFO("tenant status is abnormal, exit anyway", K(ret), K(tenant_id_));
+    } else {
+      LOG_WARN("check tenant status failed", K(ret), K(tenant_id_));
+    }
   } else if (OB_FAIL(root_service_->get_schema_service().get_tenant_schema_guard(tenant_id_, schema_guard))) {
     LOG_WARN("get tenant schema failed", K(ret), K(tenant_id_));
   } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, target_object_id_, is_index_exist))) {

src/rootserver/ddl_task/ob_drop_primary_key_task.cpp

@@ -38,15 +38,15 @@ ObDropPrimaryKeyTask::~ObDropPrimaryKeyTask()
 {
 }
 
-int ObDropPrimaryKeyTask::init(const uint64_t tenant_id, const int64_t task_id, const share::ObDDLType &ddl_type,
-    const int64_t data_table_id, const int64_t dest_table_id, const int64_t schema_version, const int64_t parallelism,
-    const int64_t consumer_group_id, const int32_t sub_task_trace_id, const obrpc::ObAlterTableArg &alter_table_arg,
-    const int64_t task_status, const int64_t snapshot_version)
+int ObDropPrimaryKeyTask::init(const ObTableSchema* src_table_schema, const ObTableSchema* dst_table_schema,
+                               const int64_t task_id, const share::ObDDLType &ddl_type, const int64_t parallelism,
+                               const int64_t consumer_group_id, const int32_t sub_task_trace_id,
+                               const obrpc::ObAlterTableArg &alter_table_arg, const uint64_t tenant_data_version,
+                               const int64_t task_status,const int64_t snapshot_version )
 {
   int ret = OB_SUCCESS;
-  if (OB_FAIL(ObTableRedefinitionTask::init(tenant_id, tenant_id, task_id, ddl_type, data_table_id,
-                                            dest_table_id, schema_version, schema_version, parallelism, consumer_group_id,
-                                            sub_task_trace_id, alter_table_arg, task_status, snapshot_version))) {
+  if (OB_FAIL(ObTableRedefinitionTask::init(src_table_schema, dst_table_schema, task_id, ddl_type, parallelism, consumer_group_id,
+                                            sub_task_trace_id, alter_table_arg, tenant_data_version, task_status, snapshot_version))) {
     LOG_WARN("fail to init ObDropPrimaryKeyTask", K(ret));
   } else {
     set_gmt_create(ObTimeUtility::current_time());

src/rootserver/ddl_task/ob_drop_primary_key_task.h

@@ -28,16 +28,15 @@ public:
   ObDropPrimaryKeyTask();
   virtual ~ObDropPrimaryKeyTask();
   int init(
-      const uint64_t tenant_id,
+      const ObTableSchema* src_table_schema,
+      const ObTableSchema* dst_table_schema,
       const int64_t task_id,
       const share::ObDDLType &ddl_type,
-      const int64_t data_table_id,
-      const int64_t dest_table_id,
-      const int64_t schema_version,
       const int64_t parallelism,
       const int64_t consumer_group_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg &alter_table_arg,
+      const uint64_t tenant_data_version,
       const int64_t task_status = share::ObDDLTaskStatus::PREPARE,
       const int64_t snapshot_version = 0);
   virtual int process() override;

src/rootserver/ddl_task/ob_index_build_task.cpp

@@ -102,7 +102,8 @@ int ObIndexSSTableBuildTask::process()
                                                            false/*use_heap_table_ddl*/,
                                                            !data_schema->is_user_hidden_table()/*use_schema_version_hint_for_src_table*/,
                                                            nullptr,
-                                                           sql_string))) {
+                                                           sql_string,
+                                                           compact_level_))) {
       LOG_WARN("fail to generate build replica sql", K(ret));
     } else if (OB_FAIL(data_schema->is_need_padding_for_generated_column(need_padding))) {
       LOG_WARN("fail to check need padding", K(ret));
@@ -146,10 +147,60 @@ int ObIndexSSTableBuildTask::process()
         LOG_WARN("ddl sim failure: create index build sstable failed", K(ret), K(tenant_id_), K(task_id_));
       } else if (OB_FAIL(user_sql_proxy->write(tenant_id_, sql_string.ptr(), affected_rows,
                   oracle_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE, &session_param, sql_exec_addr))) {
-        LOG_WARN("fail to execute build replica sql", K(ret), K(tenant_id_));
-      } else if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(tenant_id_, dest_table_id_, execution_id_, task_id_))) {
-        LOG_WARN("fail to check sstable checksum_report_finish",
-          K(ret), K(tenant_id_), K(dest_table_id_), K(execution_id_), K(task_id_));
+        if (ret == OB_SERVER_OUTOF_DISK_SPACE &&
+            data_format_version_ >= DATA_VERSION_4_3_0_0) {
+          // if version >= 4.3.0, would retry with compression.
+          // use tmp_ret to avoid ret being reset.
+          int tmp_ret = OB_SUCCESS;
+          sql_string.reuse();
+          SortCompactLevel compress_level = SORT_DEFAULT_LEVEL;
+          switch (compact_level_) {
+            case share::SORT_DEFAULT_LEVEL: {
+              compress_level = share::SORT_COMPRESSION_LEVEL;
+              break;
+            }
+            case share::SORT_COMPACT_LEVEL: {
+              compress_level = share::SORT_COMPRESSION_COMPACT_LEVEL;
+              break;
+            }
+            case share::SORT_ENCODE_LEVEL: {
+              compress_level = share::SORT_COMPRESSION_ENCODE_LEVEL;
+              break;
+            }
+            default: {
+              tmp_ret = OB_ERR_UNEXPECTED;
+              LOG_WARN("get unexpected compact level", K(tmp_ret), K(compact_level_));
+            }
+          }
+          if (tmp_ret != OB_SUCCESS) {
+          } else if (OB_SUCCESS != (tmp_ret = ObDDLUtil::generate_build_replica_sql(tenant_id_, data_table_id_,
+                                                           dest_table_id_,
+                                                           data_schema->get_schema_version(),
+                                                           snapshot_version_,
+                                                           execution_id_,
+                                                           task_id_,
+                                                           parallelism_,
+                                                           false/*use_heap_table_ddl*/,
+                                                           !data_schema->is_user_hidden_table()/*use_schema_version_hint_for_src_table*/,
+                                                           nullptr,
+                                                           sql_string,
+                                                           compress_level))) {
+            LOG_WARN("fail to generate build replica sql", K(tmp_ret));
+          } else if (OB_SUCCESS != (tmp_ret = user_sql_proxy->write(tenant_id_, sql_string.ptr(), affected_rows,
+                  oracle_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE, &session_param, sql_exec_addr))) {
+            LOG_WARN("fail to execute build replica sql", K(tmp_ret), K(tenant_id_));
+          } else {
+            ret = OB_SUCCESS;
+          }
+        } else {
+          LOG_WARN("fail to execute build replica sql", K(ret), K(tenant_id_));
+        }
+      }
+      if (OB_SUCC(ret)) {
+        if (OB_FAIL(ObCheckTabletDataComplementOp::check_finish_report_checksum(tenant_id_, dest_table_id_, execution_id_, task_id_))) {
+          LOG_WARN("fail to check sstable checksum_report_finish",
+            K(ret), K(tenant_id_), K(dest_table_id_), K(execution_id_), K(task_id_));
+        }
       }
     }
   }
@@ -199,7 +250,9 @@ ObAsyncTask *ObIndexSSTableBuildTask::deep_copy(char *buf, const int64_t buf_siz
         trace_id_,
         parallelism_,
         root_service_,
-        inner_sql_exec_addr_);
+        inner_sql_exec_addr_,
+        compact_level_,
+        data_format_version_);
     if (OB_SUCCESS != (task->set_nls_format(nls_date_format_, nls_timestamp_format_, nls_timestamp_tz_format_))) {
       task->~ObIndexSSTableBuildTask();
       task = nullptr;
@@ -214,7 +267,7 @@ ObIndexBuildTask::ObIndexBuildTask()
   : ObDDLTask(ObDDLType::DDL_CREATE_INDEX), index_table_id_(target_object_id_),
     is_unique_index_(false), is_global_index_(false), root_service_(nullptr), snapshot_held_(false),
     is_sstable_complete_task_submitted_(false), sstable_complete_request_time_(0), sstable_complete_ts_(0),
-    check_unique_snapshot_(0), complete_sstable_job_ret_code_(INT64_MAX), create_index_arg_()
+    check_unique_snapshot_(0), complete_sstable_job_ret_code_(INT64_MAX), create_index_arg_(), target_cg_cnt_(0)
 {
 
 }
@@ -354,11 +407,11 @@ int ObIndexBuildTask::init(
     const int32_t sub_task_trace_id,
     const obrpc::ObCreateIndexArg &create_index_arg,
     const int64_t parent_task_id /* = 0 */,
+    const uint64_t tenant_data_version,
     const int64_t task_status /* = TaskStatus::PREPARE */,
     const int64_t snapshot_version /* = 0 */)
 {
   int ret = OB_SUCCESS;
-  uint64_t tenant_data_format_version = 0;
   if (OB_UNLIKELY(is_inited_)) {
     ret = OB_INIT_TWICE;
     LOG_WARN("init twice", K(ret));
@@ -373,6 +426,7 @@ int ObIndexBuildTask::init(
           && OB_INVALID_ID != tenant_id
           && index_schema != nullptr
           && schema_version > 0
+          && tenant_data_version > 0
           && (task_status >= ObDDLTaskStatus::PREPARE && task_status <= ObDDLTaskStatus::SUCCESS)
           && task_id > 0))) {
     ret = OB_INVALID_ARGUMENT;
@@ -383,8 +437,6 @@ int ObIndexBuildTask::init(
   } else if (OB_ISNULL(index_schema)) {
     ret = OB_TABLE_NOT_EXIST;
     LOG_WARN("fail to get table schema", K(ret));
-  } else if (OB_FAIL(ObShareUtil::fetch_current_data_version(*GCTX.sql_proxy_, tenant_id, tenant_data_format_version))) {
-    LOG_WARN("get min data version failed", K(ret), K(tenant_id));
   } else if (OB_UNLIKELY((ObIndexArg::ADD_MLOG == create_index_arg_.index_action_type_)
       && (!index_schema->is_mlog_table()))) {
     ret = OB_ERR_UNEXPECTED;
@@ -413,7 +465,7 @@ int ObIndexBuildTask::init(
     parent_task_id_ = parent_task_id;
     task_version_ = OB_INDEX_BUILD_TASK_VERSION;
     start_time_ = ObTimeUtility::current_time();
-    data_format_version_ = tenant_data_format_version;
+    data_format_version_ = tenant_data_version;
     if (OB_SUCC(ret)) {
       task_status_ = static_cast<ObDDLTaskStatus>(task_status);
     }
@@ -425,8 +477,14 @@ int ObIndexBuildTask::init(
       dst_schema_version_ = schema_version_;
       is_inited_ = true;
     }
+
+    if (OB_FAIL(ret)) {
+    } else if (OB_FAIL(index_schema->get_store_column_group_count(target_cg_cnt_))) {
+      LOG_WARN("fail to get column group cnt", K(ret), K(index_schema));
+    }
     ddl_tracing_.open();
   }
+
   return ret;
 }
 
@@ -864,7 +922,9 @@ int ObIndexBuildTask::send_build_single_replica_request()
         trace_id_,
         parallelism_,
         root_service_,
-        create_index_arg_.inner_sql_exec_addr_);
+        create_index_arg_.inner_sql_exec_addr_,
+        create_index_arg_.compact_level_,
+        data_format_version_);
     if (OB_FAIL(task.set_nls_format(create_index_arg_.nls_date_format_,
                                     create_index_arg_.nls_timestamp_format_,
                                     create_index_arg_.nls_timestamp_tz_format_))) {
@@ -1507,17 +1567,36 @@ int ObIndexBuildTask::collect_longops_stat(ObLongopsValue &value)
     case ObDDLTaskStatus::REDEFINITION: {
       int64_t row_scanned = 0;
       int64_t row_sorted = 0;
-      int64_t row_inserted = 0;
-      if (OB_FAIL(gather_redefinition_stats(tenant_id_, task_id_, *GCTX.sql_proxy_, row_scanned, row_sorted, row_inserted))) {
+      int64_t row_inserted_cg = 0;
+      int64_t row_inserted_file = 0;
+
+      if (OB_FAIL(gather_redefinition_stats(tenant_id_, task_id_, *GCTX.sql_proxy_, row_scanned, row_sorted, row_inserted_cg, row_inserted_file))) {
         LOG_WARN("failed to gather redefinition stats", K(ret));
-      } else if (OB_FAIL(databuff_printf(stat_info_.message_,
-                                  MAX_LONG_OPS_MESSAGE_LENGTH,
-                                  pos,
-                                  "STATUS: REPLICA BUILD, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED: %ld",
-                                  row_scanned,
-                                  row_sorted,
-                                  row_inserted))) {
-        LOG_WARN("failed to print", K(ret));
+      }
+
+      if (OB_FAIL(ret)){
+      } else if (target_cg_cnt_ > 1) {
+        if (OB_FAIL(databuff_printf(stat_info_.message_,
+                                    MAX_LONG_OPS_MESSAGE_LENGTH,
+                                    pos,
+                                    "STATUS: REPLICA BUILD, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED_INTO_TMP_FILE: %ld, ROW_INSERTED: %ld out of %ld column group rows",
+                                    row_scanned,
+                                    row_sorted,
+                                    row_inserted_file,
+                                    row_inserted_cg,
+                                    row_scanned * target_cg_cnt_))) {
+          LOG_WARN("failed to print", K(ret));
+        }
+      } else {
+        if (OB_FAIL(databuff_printf(stat_info_.message_,
+                                    MAX_LONG_OPS_MESSAGE_LENGTH,
+                                    pos,
+                                    "STATUS: REPLICA BUILD, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED: %ld",
+                                    row_scanned,
+                                    row_sorted,
+                                    row_inserted_file))) {
+          LOG_WARN("failed to print", K(ret));
+        }
       }
       break;
     }
@@ -1583,7 +1662,7 @@ int ObIndexBuildTask::serialize_params_to_message(char *buf, const int64_t buf_l
   } else if (OB_FAIL(create_index_arg_.serialize(buf, buf_len, pos))) {
     LOG_WARN("serialize create index arg failed", K(ret));
   } else {
-    LST_DO_CODE(OB_UNIS_ENCODE, check_unique_snapshot_);
+    LST_DO_CODE(OB_UNIS_ENCODE, check_unique_snapshot_, target_cg_cnt_);
   }
   return ret;
 }
@@ -1604,7 +1683,7 @@ int ObIndexBuildTask::deserlize_params_from_message(const uint64_t tenant_id, co
   } else if (OB_FAIL(deep_copy_table_arg(allocator_, tmp_arg, create_index_arg_))) {
     LOG_WARN("deep copy create index arg failed", K(ret));
   } else {
-    LST_DO_CODE(OB_UNIS_DECODE, check_unique_snapshot_);
+    LST_DO_CODE(OB_UNIS_DECODE, check_unique_snapshot_, target_cg_cnt_);
   }
   return ret;
 }
@@ -1613,5 +1692,6 @@ int64_t ObIndexBuildTask::get_serialize_param_size() const
 {
   return create_index_arg_.get_serialize_size()
       + serialization::encoded_length_i64(check_unique_snapshot_)
-      + ObDDLTask::get_serialize_param_size();
+      + ObDDLTask::get_serialize_param_size()
+      + serialization::encoded_length_i64(target_cg_cnt_);
 }

src/rootserver/ddl_task/ob_index_build_task.h

@@ -35,11 +35,14 @@ public:
       const common::ObCurTraceId::TraceId &trace_id,
       const int64_t parallelism,
       ObRootService *root_service,
-      const common::ObAddr &inner_sql_exec_addr)
+      const common::ObAddr &inner_sql_exec_addr,
+      const share::SortCompactLevel compact_level = share::SORT_DEFAULT_LEVEL,
+      const int64_t data_format_version = 0)
       : task_id_(task_id), tenant_id_(tenant_id), data_table_id_(data_table_id), dest_table_id_(dest_table_id),
         schema_version_(schema_version), snapshot_version_(snapshot_version), execution_id_(execution_id),
         consumer_group_id_(consumer_group_id), trace_id_(trace_id), parallelism_(parallelism), allocator_("IdxSSTBuildTask"),
-        root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr)
+        root_service_(root_service), inner_sql_exec_addr_(inner_sql_exec_addr), compact_level_(compact_level),
+        data_format_version_(data_format_version)
   {
     set_retry_times(0);
   }
@@ -55,7 +58,7 @@ public:
   void add_event_info(const int ret, const ObString &ddl_event_stmt);
   TO_STRING_KV(K_(data_table_id), K_(dest_table_id), K_(schema_version), K_(snapshot_version),
                K_(execution_id), K_(consumer_group_id), K_(trace_id), K_(parallelism), K_(nls_date_format),
-               K_(nls_timestamp_format), K_(nls_timestamp_tz_format));
+               K_(nls_timestamp_format), K_(nls_timestamp_tz_format), K_(compact_level), K_(data_format_version));
 
 private:
   int64_t task_id_;
@@ -74,6 +77,8 @@ private:
   ObString nls_timestamp_tz_format_;
   ObRootService *root_service_;
   common::ObAddr inner_sql_exec_addr_;
+  share::SortCompactLevel compact_level_;
+  int64_t data_format_version_;
 
   DISALLOW_COPY_AND_ASSIGN(ObIndexSSTableBuildTask);
 };
@@ -95,6 +100,7 @@ public:
       const int32_t sub_task_trace_id,
       const obrpc::ObCreateIndexArg &create_index_arg,
       const int64_t parent_task_id /* = 0 */,
+      const uint64_t tenant_data_version,
       const int64_t task_status = share::ObDDLTaskStatus::PREPARE,
       const int64_t snapshot_version = 0);
   int init(const ObDDLTaskRecord &task_record);
@@ -119,7 +125,7 @@ public:
   virtual bool support_longops_monitoring() const override { return true; }
   static int deep_copy_index_arg(common::ObIAllocator &allocator, const obrpc::ObCreateIndexArg &source_arg, obrpc::ObCreateIndexArg &dest_arg);
   INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask, K(index_table_id_),K(snapshot_held_), K(is_sstable_complete_task_submitted_),
-      K(sstable_complete_ts_), K(check_unique_snapshot_), K_(redefinition_execution_id), K(create_index_arg_));
+      K(sstable_complete_ts_), K(check_unique_snapshot_), K_(redefinition_execution_id), K(create_index_arg_), K(target_cg_cnt_));
 private:
   int prepare();
   int wait_trans_end();
@@ -142,6 +148,7 @@ private:
                                       const ObTableSchema *index_table_schema,
                                       bool &need_acquire);
   bool is_sstable_complete_task_submitted();
+  int check_target_cg_cnt();
 private:
   static const int64_t OB_INDEX_BUILD_TASK_VERSION = 1;
   using ObDDLTask::is_inited_;
@@ -164,6 +171,7 @@ private:
   int64_t complete_sstable_job_ret_code_;
   int64_t redefinition_execution_id_;
   obrpc::ObCreateIndexArg create_index_arg_; // this is not a valid arg, only has nls formats for now
+  int64_t target_cg_cnt_;
 };
 
 }  // end namespace rootserver

src/rootserver/ddl_task/ob_recover_restore_table_task.cpp

@@ -37,20 +37,30 @@ ObRecoverRestoreTableTask::~ObRecoverRestoreTableTask()
 {
 }
 
-int ObRecoverRestoreTableTask::init(const uint64_t src_tenant_id, const uint64_t dst_tenant_id, const int64_t task_id,
-    const share::ObDDLType &ddl_type, const int64_t data_table_id, const int64_t dest_table_id, const int64_t src_schema_version,
-    const int64_t dst_schema_version, const int64_t parallelism, const int64_t consumer_group_id, const int32_t sub_task_trace_id,
-    const ObAlterTableArg &alter_table_arg, const int64_t task_status, const int64_t snapshot_version)
+int ObRecoverRestoreTableTask::init(
+    const ObTableSchema* src_table_schema, const ObTableSchema* dst_table_schema,
+    const int64_t task_id, const share::ObDDLType &ddl_type, const int64_t parallelism,
+    const int64_t consumer_group_id, const int32_t sub_task_trace_id,
+    const obrpc::ObAlterTableArg &alter_table_arg, const uint64_t tenant_data_version, const int64_t task_status, const int64_t snapshot_version)
 {
   int ret = OB_SUCCESS;
   if (OB_UNLIKELY(is_inited_)) {
     ret = OB_INIT_TWICE;
     LOG_WARN("init twice", K(ret));
+  } else if (OB_ISNULL(src_table_schema)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("src schema should not be null", K(ret));
+  } else if (OB_ISNULL(dst_table_schema)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("dst_table schema should not be null", K(ret));
+  } else if ((!src_table_schema->is_valid()) || (!dst_table_schema->is_valid())) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("src_talbe or dst_table is invalid", K(ret), KPC(src_table_schema), KPC(dst_table_schema));
   } else if (OB_UNLIKELY(ObDDLType::DDL_TABLE_RESTORE != ddl_type)) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid arg", K(ret), K(ddl_type), K(src_tenant_id), K(data_table_id));
-  } else if (OB_FAIL(ObTableRedefinitionTask::init(src_tenant_id, dst_tenant_id, task_id, ddl_type, data_table_id,
-      dest_table_id, src_schema_version, dst_schema_version, parallelism, consumer_group_id, sub_task_trace_id, alter_table_arg, task_status, 0/*snapshot*/))) {
+    LOG_WARN("invalid arg", K(ret), K(ddl_type), KPC(src_table_schema), KPC(dst_table_schema));
+  } else if (OB_FAIL(ObTableRedefinitionTask::init(src_table_schema, dst_table_schema, task_id, ddl_type, parallelism, consumer_group_id,
+                                                   sub_task_trace_id, alter_table_arg, tenant_data_version, task_status, 0/*snapshot*/))) {
     LOG_WARN("fail to init ObDropPrimaryKeyTask", K(ret));
   } else {
     execution_id_ = 1L;

src/rootserver/ddl_task/ob_recover_restore_table_task.h

@@ -32,18 +32,15 @@ public:
   ObRecoverRestoreTableTask();
   virtual ~ObRecoverRestoreTableTask();
   int init(
-      const uint64_t src_tenant_id,
-      const uint64_t dst_tenant_id,
+      const ObTableSchema* src_table_schema,
+      const ObTableSchema* dst_table_schema,
       const int64_t task_id,
       const share::ObDDLType &ddl_type,
-      const int64_t data_table_id,
-      const int64_t dest_table_id,
-      const int64_t src_schema_version,
-      const int64_t dest_schema_version,
       const int64_t parallelism,
       const int64_t consumer_group_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg &alter_table_arg,
+      const uint64_t tenant_data_version,
       const int64_t task_status = share::ObDDLTaskStatus::PREPARE,
       const int64_t snapshot_version = 0);
   int init(const ObDDLTaskRecord &task_record);

src/rootserver/ddl_task/ob_table_redefinition_task.cpp

@@ -39,7 +39,7 @@ ObTableRedefinitionTask::ObTableRedefinitionTask()
     has_rebuild_index_(false), has_rebuild_constraint_(false), has_rebuild_foreign_key_(false),
     allocator_(lib::ObLabel("RedefTask")),
     is_copy_indexes_(true), is_copy_triggers_(true), is_copy_constraints_(true), is_copy_foreign_keys_(true),
-    is_ignore_errors_(false), is_do_finish_(false)
+    is_ignore_errors_(false), is_do_finish_(false), target_cg_cnt_(0)
 {
 }
 
@@ -47,61 +47,69 @@ ObTableRedefinitionTask::~ObTableRedefinitionTask()
 {
 }
 
-int ObTableRedefinitionTask::init(const uint64_t src_tenant_id, const uint64_t dst_tenant_id, const int64_t task_id,
-    const share::ObDDLType &ddl_type, const int64_t data_table_id, const int64_t dest_table_id, const int64_t src_schema_version,
-    const int64_t dst_schema_version, const int64_t parallelism, const int64_t consumer_group_id, const int32_t sub_task_trace_id,
-    const ObAlterTableArg &alter_table_arg, const int64_t task_status, const int64_t snapshot_version)
+int ObTableRedefinitionTask::init(const ObTableSchema* src_table_schema, const ObTableSchema* dst_table_schema, const int64_t task_id,
+    const share::ObDDLType &ddl_type, const int64_t parallelism, const int64_t consumer_group_id, const int32_t sub_task_trace_id,
+    const ObAlterTableArg &alter_table_arg, const uint64_t tenant_data_version, const int64_t task_status, const int64_t snapshot_version)
 {
   int ret = OB_SUCCESS;
-  uint64_t tenant_data_format_version = 0;
   if (OB_UNLIKELY(is_inited_)) {
     ret = OB_INIT_TWICE;
     LOG_WARN("ObTableRedefinitionTask has already been inited", K(ret));
-  } else if (OB_UNLIKELY(OB_INVALID_ID == src_tenant_id || OB_INVALID_ID == dst_tenant_id
-    || task_id <= 0 || OB_INVALID_ID == data_table_id || OB_INVALID_ID == dest_table_id
-    || src_schema_version <= 0 || dst_schema_version <= 0
-    || task_status < ObDDLTaskStatus::PREPARE || task_status > ObDDLTaskStatus::SUCCESS || snapshot_version < 0
-    || (snapshot_version > 0 && task_status < ObDDLTaskStatus::WAIT_TRANS_END))) {
+  } else if (OB_ISNULL(src_table_schema) || OB_ISNULL(dst_table_schema)) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid arguments", K(ret), K(src_tenant_id), K(dst_tenant_id), K(task_id),
-                                  K(data_table_id), K(dest_table_id), K(src_schema_version), K(dst_schema_version),
+    LOG_WARN("invalid argument", K(ret), KP(src_table_schema), KP(dst_table_schema));
+  } else if (OB_UNLIKELY( !src_table_schema->is_valid()
+                        || !dst_table_schema->is_valid()
+                        || task_id <= 0  || snapshot_version < 0 || tenant_data_version <= 0
+                        || task_status < ObDDLTaskStatus::PREPARE || task_status > ObDDLTaskStatus::SUCCESS
+                        || (snapshot_version > 0 && task_status < ObDDLTaskStatus::WAIT_TRANS_END))) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid arguments", K(ret), KPC(src_table_schema), KPC(dst_table_schema), K(task_id),
                                   K(task_status), K(snapshot_version));
   } else if (OB_FAIL(deep_copy_table_arg(allocator_, alter_table_arg, alter_table_arg_))) {
     LOG_WARN("deep copy alter table arg failed", K(ret));
   } else if (OB_FAIL(set_ddl_stmt_str(alter_table_arg_.ddl_stmt_str_))) {
     LOG_WARN("set ddl stmt str failed", K(ret));
-  } else if (OB_FAIL(ObShareUtil::fetch_current_data_version(*GCTX.sql_proxy_, src_tenant_id, tenant_data_format_version))) {
-    LOG_WARN("get min data version failed", K(ret), K(src_tenant_id));
   } else {
     set_gmt_create(ObTimeUtility::current_time());
     consumer_group_id_ = consumer_group_id;
     sub_task_trace_id_ = sub_task_trace_id;
     task_type_ = ddl_type;
-    object_id_ = data_table_id;
-    target_object_id_ = dest_table_id;
-    schema_version_ = src_schema_version;
+    object_id_ = src_table_schema->get_table_id();
+    target_object_id_ = dst_table_schema->get_table_id();
+
+    /* only table restore set schema_serson = src, other use dst*/
+    if (ObDDLType::DDL_TABLE_RESTORE == ddl_type) {
+      schema_version_ = src_table_schema->get_schema_version();
+    } else {
+      schema_version_ = dst_table_schema->get_schema_version();
+    }
+
     task_status_ = static_cast<ObDDLTaskStatus>(task_status);
     snapshot_version_ = snapshot_version;
-    tenant_id_ = src_tenant_id;
+    tenant_id_ = src_table_schema->get_tenant_id();
     task_version_ = OB_TABLE_REDEFINITION_TASK_VERSION;
     task_id_ = task_id;
     parallelism_ = parallelism;
-    data_format_version_ = tenant_data_format_version;
+    data_format_version_ = tenant_data_version;
     start_time_ = ObTimeUtility::current_time();
     // For common offline ddl, dest_tenant_id is also the tenant_id_, i.e., tenant id of the data table.
     // But for DDL_RESTORE_TABLE, dst_tenant_id_ is different to the tenant_id_.
-    dst_tenant_id_ = dst_tenant_id;
-    dst_schema_version_ = dst_schema_version;
-    alter_table_arg_.alter_table_schema_.set_tenant_id(src_tenant_id);
-    alter_table_arg_.alter_table_schema_.set_schema_version(src_schema_version);
+    dst_tenant_id_ = dst_table_schema->get_tenant_id();
+    dst_schema_version_ = dst_table_schema->get_schema_version();
+    alter_table_arg_.alter_table_schema_.set_tenant_id(tenant_id_);
+    alter_table_arg_.alter_table_schema_.set_schema_version(schema_version_);
     alter_table_arg_.exec_tenant_id_ = dst_tenant_id_;
-    if (OB_FAIL(init_ddl_task_monitor_info(target_object_id_))) {
+    if (OB_FAIL(dst_table_schema->get_store_column_group_count(target_cg_cnt_))) {
+      LOG_WARN("fail to get target cg cnt", K(ret), KPC(dst_table_schema));
+    } else if (OB_FAIL(init_ddl_task_monitor_info(target_object_id_))) {
       LOG_WARN("init ddl task monitor info failed", K(ret));
     } else {
       is_inited_ = true;
       ddl_tracing_.open();
     }
   }
+
   LOG_INFO("init table redefinition task finished", K(ret), KPC(this));
   return ret;
 }
@@ -167,6 +175,7 @@ int ObTableRedefinitionTask::init(const ObDDLTaskRecord &task_record)
       ddl_tracing_.open_for_recovery();
     }
   }
+
   LOG_INFO("init table redefinition task finished", K(ret), KPC(this));
   return ret;
 }
@@ -283,7 +292,8 @@ int ObTableRedefinitionTask::send_build_replica_request_by_sql()
         alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_,
         alter_table_arg_.mview_refresh_info_.mview_table_id_,
         GCTX.root_service_,
-        alter_table_arg_.inner_sql_exec_addr_);
+        alter_table_arg_.inner_sql_exec_addr_,
+        data_format_version_);
     if (OB_FAIL(root_service->get_ddl_service().get_tenant_schema_guard_with_version_in_inner_table(tenant_id_, schema_guard))) {
       LOG_WARN("get schema guard failed", K(ret));
     } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, object_id_, orig_table_schema))) {
@@ -531,6 +541,7 @@ int ObTableRedefinitionTask::copy_table_indexes()
                                          &create_index_arg,
                                          task_id_);
               param.sub_task_trace_id_ = sub_task_trace_id_;
+              param.tenant_data_version_ = data_format_version_;
               if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, *GCTX.sql_proxy_, task_record))) {
                 if (OB_ENTRY_EXIST == ret) {
                   ret = OB_SUCCESS;
@@ -1037,7 +1048,8 @@ int64_t ObTableRedefinitionTask::get_serialize_param_size() const
   return alter_table_arg_.get_serialize_size() + ObDDLTask::get_serialize_param_size()
          + serialization::encoded_length_i8(copy_indexes) + serialization::encoded_length_i8(copy_triggers)
          + serialization::encoded_length_i8(copy_constraints) + serialization::encoded_length_i8(copy_foreign_keys)
-         + serialization::encoded_length_i8(ignore_errors) + serialization::encoded_length_i8(do_finish);
+         + serialization::encoded_length_i8(ignore_errors) + serialization::encoded_length_i8(do_finish)
+         + serialization::encoded_length_i64(target_cg_cnt_);
 }
 
 int ObTableRedefinitionTask::serialize_params_to_message(char *buf, const int64_t buf_len, int64_t &pos) const
@@ -1068,6 +1080,8 @@ int ObTableRedefinitionTask::serialize_params_to_message(char *buf, const int64_
     LOG_WARN("fail to serialize is_ignore_errors", K(ret));
   } else if (OB_FAIL(serialization::encode_i8(buf, buf_len, pos, do_finish))) {
     LOG_WARN("fail to serialize is_do_finish", K(ret));
+  } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, target_cg_cnt_))) {
+    LOG_WARN("fail to serialize target_cg_cnt", K(ret));
   }
   FLOG_INFO("serialize message for table redefinition", K(ret),
       K(copy_indexes), K(copy_triggers), K(copy_constraints), K(copy_foreign_keys), K(ignore_errors), K(do_finish), K(*this));
@@ -1108,7 +1122,10 @@ int ObTableRedefinitionTask::deserlize_params_from_message(const uint64_t tenant
       LOG_WARN("fail to deserialize is_ignore_errors_", K(ret));
     } else if (OB_FAIL(serialization::decode_i8(buf, data_len, pos, &do_finish))) {
       LOG_WARN("fail to deserialize is_do_finish_", K(ret));
-    } else {
+    } else if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &target_cg_cnt_))) {
+      LOG_WARN("fail to deserialize target_cg_ctn_", K(ret));
+    }
+    else {
       is_copy_indexes_ = static_cast<bool>(copy_indexes);
       is_copy_triggers_ = static_cast<bool>(copy_triggers);
       is_copy_constraints_ = static_cast<bool>(copy_constraints);
@@ -1197,17 +1214,37 @@ int ObTableRedefinitionTask::collect_longops_stat(ObLongopsValue &value)
     case ObDDLTaskStatus::REDEFINITION: {
       int64_t row_scanned = 0;
       int64_t row_sorted = 0;
-      int64_t row_inserted = 0;
-      if (OB_FAIL(gather_redefinition_stats(dst_tenant_id_, task_id_, *GCTX.sql_proxy_, row_scanned, row_sorted, row_inserted))) {
+      int64_t row_inserted_cg = 0;
+      int64_t row_inserted_file = 0;
+
+      if (OB_FAIL(gather_redefinition_stats(dst_tenant_id_, task_id_, *GCTX.sql_proxy_, row_scanned, row_sorted, row_inserted_cg, row_inserted_file))) {
         LOG_WARN("failed to gather redefinition stats", K(ret));
-      } else if (OB_FAIL(databuff_printf(stat_info_.message_,
-                                  MAX_LONG_OPS_MESSAGE_LENGTH,
-                                  pos,
-                                  "STATUS: REPLICA BUILD, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED: %ld",
-                                  row_scanned,
-                                  row_sorted,
-                                  row_inserted))) {
-        LOG_WARN("failed to print", K(ret));
+      }
+
+
+      if (OB_FAIL(ret)){
+      } else if (target_cg_cnt_> 1) {
+        if (OB_FAIL(databuff_printf(stat_info_.message_,
+                                    MAX_LONG_OPS_MESSAGE_LENGTH,
+                                    pos,
+                                    "STATUS: REPLICA BUILD, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED_TMP_FILE: %ld, ROW_INSERTED: %ld out of %ld column group rows",
+                                    row_scanned,
+                                    row_sorted,
+                                    row_inserted_file,
+                                    row_inserted_cg,
+                                    row_scanned * target_cg_cnt_))) {
+          LOG_WARN("failed to print", K(ret));
+        }
+      } else {
+        if (OB_FAIL(databuff_printf(stat_info_.message_,
+                                    MAX_LONG_OPS_MESSAGE_LENGTH,
+                                    pos,
+                                    "STATUS: REPLICA BUILD, ROW_SCANNED: %ld, ROW_SORTED: %ld, ROW_INSERTED: %ld",
+                                    row_scanned,
+                                    row_sorted,
+                                    row_inserted_file))) {
+          LOG_WARN("failed to print", K(ret));
+        }
       }
       break;
     }

src/rootserver/ddl_task/ob_table_redefinition_task.h

@@ -32,18 +32,15 @@ public:
   ObTableRedefinitionTask();
   virtual ~ObTableRedefinitionTask();
   int init(
-      const uint64_t tenant_id,
-      const uint64_t dest_tenant_id,
+      const ObTableSchema* src_table_schema,
+      const ObTableSchema* dst_table_schema,
       const int64_t task_id,
       const share::ObDDLType &ddl_type,
-      const int64_t data_table_id,
-      const int64_t dest_table_id,
-      const int64_t schema_version,
-      const int64_t dest_schema_version,
       const int64_t parallelism,
       const int64_t consumer_group_id,
       const int32_t sub_task_trace_id,
       const obrpc::ObAlterTableArg &alter_table_arg,
+      const uint64_t tenant_data_version,
       const int64_t task_status = share::ObDDLTaskStatus::PREPARE,
       const int64_t snapshot_version = 0);
   int init(const ObDDLTaskRecord &task_record);
@@ -72,7 +69,7 @@ public:
   INHERIT_TO_STRING_KV("ObDDLRedefinitionTask", ObDDLRedefinitionTask,
       K(has_rebuild_index_), K(has_rebuild_constraint_), K(has_rebuild_foreign_key_),
       K(is_copy_indexes_), K(is_copy_triggers_), K(is_copy_constraints_),
-      K(is_copy_foreign_keys_), K(is_ignore_errors_), K(is_do_finish_));
+      K(is_copy_foreign_keys_), K(is_ignore_errors_), K(is_do_finish_), K(target_cg_cnt_));
 protected:
   int table_redefinition(const share::ObDDLTaskStatus next_task_status);
   int copy_table_dependent_objects(const share::ObDDLTaskStatus next_task_status);
@@ -99,6 +96,7 @@ private:
   int check_modify_autoinc(bool &modify_autoinc);
   int check_use_heap_table_ddl_plan(bool &use_heap_table_ddl_plan);
   int get_direct_load_job_stat(common::ObArenaAllocator &allocator, sql::ObLoadDataStat &job_stat);
+  int check_target_cg_cnt();
 private:
   static const int64_t OB_TABLE_REDEFINITION_TASK_VERSION = 1L;
   bool has_rebuild_index_;
@@ -111,6 +109,7 @@ private:
   bool is_copy_foreign_keys_;
   bool is_ignore_errors_;
   bool is_do_finish_;
+  int64_t target_cg_cnt_;
 };
 
 }  // end namespace rootserver

src/rootserver/ob_bootstrap.cpp

@@ -679,16 +679,19 @@ int ObBootstrap::prepare_create_partition(
     common::ObArray<share::schema::ObTableSchema> table_schema_array;
     common::ObArray<const share::schema::ObTableSchema*> table_schema_ptrs;
     common::ObArray<share::ObLSID> ls_id_array;
+    common::ObArray<bool> need_create_empty_majors;
     if (OB_FAIL(generate_table_schema_array_for_create_partition(tschema, table_schema_array))) {
       LOG_WARN("fail to generate table schema array", KR(ret));
     } else if (OB_UNLIKELY(table_schema_array.count() < 1)) {
       ret = OB_ERR_UNEXPECTED;
       LOG_WARN("generate table schema count is unexpected", KR(ret));
-    } else if (OB_FAIL(table_schema_ptrs.reserve(table_schema_array.count()))) {
+    } else if (OB_FAIL(table_schema_ptrs.reserve(table_schema_array.count()))
+      || OB_FAIL(need_create_empty_majors.reserve(table_schema_array.count()))) {
       LOG_WARN("Fail to reserve rowkey column array", KR(ret));
     } else {
       for (int i = 0; i < table_schema_array.count() && OB_SUCC(ret); ++i) {
-        if (OB_FAIL(table_schema_ptrs.push_back(&table_schema_array.at(i)))) {
+        if (OB_FAIL(table_schema_ptrs.push_back(&table_schema_array.at(i)))
+          || OB_FAIL(need_create_empty_majors.push_back(true))) {
           LOG_WARN("fail to push back", KR(ret), K(table_schema_array));
         }
       }
@@ -702,7 +705,9 @@ int ObBootstrap::prepare_create_partition(
     if (OB_FAIL(ret)) {
     } else if (OB_FAIL(creator.add_create_tablets_of_tables_arg(
             table_schema_ptrs,
-            ls_id_array))) {
+            ls_id_array,
+            DATA_CURRENT_VERSION,
+            need_create_empty_majors/*need_create_empty_major_sstable*/))) {
       LOG_WARN("fail to add create tablet arg", KR(ret));
     }
   }

src/rootserver/ob_ddl_operator.cpp

@@ -3998,10 +3998,69 @@ int ObDDLOperator::update_single_column(common::ObMySQLTransaction &trans,
     LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id));
   } else {
     column_schema.set_schema_version(new_schema_version);
+    const ObColumnSchemaV2 *orig_column_schema = origin_table_schema.get_column_schema(column_schema.get_column_id());
     if (OB_FAIL(schema_service_impl->get_table_sql_service().update_single_column(
               trans, origin_table_schema, new_table_schema, column_schema,
               true /* record_ddl_operation */))) {
       RS_LOG(WARN, "failed to update single column", K(ret));
+    } else if (OB_ISNULL(orig_column_schema)) {
+      ret = OB_ERR_UNEXPECTED;
+      RS_LOG(WARN, "failed to get orig column schema", K(ret), K(origin_table_schema), K(column_schema));
+    } else if (OB_FAIL(update_single_column_group(trans, origin_table_schema, *orig_column_schema, column_schema))) {
+      RS_LOG(WARN, "fail to update single column group", K(ret));
+    }
+  }
+  return ret;
+}
+
+int ObDDLOperator::update_single_column_group(common::ObMySQLTransaction &trans,
+                                              const ObTableSchema &origin_table_schema,
+                                              const ObColumnSchemaV2 &origin_column_schema,
+                                              const ObColumnSchemaV2 &column_schema)
+{
+  int ret = OB_SUCCESS;
+  bool is_each_cg_exist = false;
+  char cg_name[OB_MAX_COLUMN_GROUP_NAME_LENGTH] = {'\0'};
+  ObString cg_name_str(OB_MAX_COLUMN_GROUP_NAME_LENGTH, 0, cg_name);
+  const uint64_t tenant_id = origin_table_schema.get_tenant_id();
+  ObColumnGroupSchema *ori_cg = nullptr;
+  ObSchemaService *schema_service_impl = schema_service_.get_schema_service();
+  if (!origin_table_schema.is_valid() || !origin_column_schema.is_valid() || !column_schema.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    RS_LOG(WARN, "Invalid arguemnt", K(ret), K(origin_table_schema), K(origin_column_schema), K(column_schema));
+  } else if (origin_column_schema.get_column_name_str() == column_schema.get_column_name_str()) {
+    /* now only rename column will use this func, other skip*/
+  } else if (!origin_table_schema.is_column_store_supported()) {
+    /* only support table need column group*/
+  } else if (OB_FAIL(origin_table_schema.is_column_group_exist(OB_EACH_COLUMN_GROUP_NAME, is_each_cg_exist))) {
+    RS_LOG(WARN, "fail check whether each cg exist", K(ret));
+  } else if (!is_each_cg_exist) {
+    /* if each cg not exist skip*/
+  } else if (OB_FAIL(origin_column_schema.get_each_column_group_name(cg_name_str))) {
+    RS_LOG(WARN, "fail to get each column group name", K(ret));
+  } else if (OB_FAIL(origin_table_schema.get_column_group_by_name(cg_name_str, ori_cg))) {
+    RS_LOG(WARN, "column group cannot get", K(cg_name_str), K(origin_table_schema));
+  } else if (OB_ISNULL(ori_cg)) {
+    ret = OB_ERR_UNEXPECTED;
+    RS_LOG(WARN, "column group should not be null", K(ret), K(cg_name_str),
+           K(origin_column_schema), K(origin_table_schema));
+  } else {
+    ObColumnGroupSchema new_cg;
+    if (OB_FAIL(new_cg.assign(*ori_cg))) {
+      RS_LOG(WARN, "fail to assign column group", K(ret), K(ori_cg));
+    } else {
+      new_cg.set_schema_version(column_schema.get_schema_version());
+      cg_name_str.set_length(0);
+      if (OB_FAIL(column_schema.get_each_column_group_name(cg_name_str))) {
+        RS_LOG(WARN, "fail to gen column group related column group name", K(ret), K(column_schema));
+      } else if (OB_FAIL(new_cg.set_column_group_name(cg_name_str))) {
+        RS_LOG(WARN, "fail to set column group name", K(ret), K(new_cg), K(cg_name_str));
+      } else if (OB_FAIL(schema_service_impl->get_table_sql_service().update_single_column_group(trans,
+                                                                                origin_table_schema,
+                                                                                *ori_cg,
+                                                                                new_cg))) {
+        RS_LOG(WARN,"fail to update single column_group", K(ret));
+      }
     }
   }
   return ret;

src/rootserver/ob_ddl_operator.h

@@ -292,10 +292,10 @@ public:
                                 const share::schema::ObTableSchema &inc_table_schema,
                                 common::ObIArray<share::schema::ObPartition*> &part_array);
   int insert_column_groups(ObMySQLTransaction &trans, const ObTableSchema &new_table_schema);
-	int insert_column_ids_into_column_group(ObMySQLTransaction &trans,
-																				 const ObTableSchema &new_table_schema,
-																				 const ObIArray<uint64_t> &column_ids,
-																				 const ObColumnGroupSchema &column_group);
+  int insert_column_ids_into_column_group(ObMySQLTransaction &trans,
+                                          const ObTableSchema &new_table_schema,
+                                          const ObIArray<uint64_t> &column_ids,
+                                          const ObColumnGroupSchema &column_group);
   int insert_single_column(common::ObMySQLTransaction &trans,
                            const share::schema::ObTableSchema &new_table_schema,
                            share::schema::ObColumnSchemaV2 &new_column);
@@ -977,6 +977,10 @@ public:
                            const share::schema::ObTableSchema &origin_table_schema,
                            const share::schema::ObTableSchema &new_table_schema,
                            share::schema::ObColumnSchemaV2 &column_schema);
+  int update_single_column_group(common::ObMySQLTransaction &trans,
+                                 const ObTableSchema &origin_table_schema,
+                                 const ObColumnSchemaV2 &origin_column_schema,
+                                 const ObColumnSchemaV2 &new_column_schema);
   int update_partition_option(common::ObMySQLTransaction &trans,
                               share::schema::ObTableSchema &table_schema);
   int update_check_constraint_state(common::ObMySQLTransaction &trans,

src/rootserver/ob_ddl_service.h

@@ -134,18 +134,21 @@ public:
 
   // create_index_table will fill table_id and frozen_version to table_schema
   virtual int create_index_table(const obrpc::ObCreateIndexArg &arg,
+                                 const uint64_t tenant_data_version,
                                  share::schema::ObTableSchema &table_schema,
                                  ObMySQLTransaction &sql_trans);
 
   virtual int create_mlog_table(ObMySQLTransaction &sql_trans,
                                 const obrpc::ObCreateMLogArg &arg,
+                                const uint64_t tenant_data_version,
                                 share::schema::ObSchemaGetterGuard &schema_guard,
                                 share::schema::ObTableSchema &table_schema);
 
   virtual int create_mlog_tablet(ObMySQLTransaction &trans,
                                  share::schema::ObSchemaGetterGuard &schema_guard,
                                  const share::schema::ObTableSchema &mlog_schema,
-                                 const bool need_check_tablet_cnt);
+                                 const bool need_check_tablet_cnt,
+                                 const uint64_t tenant_data_version);
 
   virtual int add_mlog(ObMySQLTransaction &trans,
                        const obrpc::ObCreateMLogArg &arg,
@@ -159,10 +162,12 @@ public:
                              const share::schema::ObTableSchema &data_table_schema,
                              share::schema::ObTableSchema &table_schema,
                              const ObString *ddl_stmt_str,
-                             ObMySQLTransaction *sql_trans);
+                             ObMySQLTransaction *sql_trans,
+                             const uint64_t tenant_data_version);
 
   int create_inner_expr_index(ObMySQLTransaction &trans,
                               const share::schema::ObTableSchema &orig_table_schema,
+                              const uint64_t tenant_data_version,
                               share::schema::ObTableSchema &new_table_schema,
                               common::ObIArray<share::schema::ObColumnSchemaV2*> &new_columns,
                               share::schema::ObTableSchema &index_schema);
@@ -175,10 +180,12 @@ public:
       ObMySQLTransaction &trans,
       const obrpc::ObCreateIndexArg &arg,
       const share::schema::ObTableSchema &table_schema,
+      const uint64_t tenant_data_version,
       share::schema::ObTableSchema &index_schema);
   int create_global_inner_expr_index(
       ObMySQLTransaction &trans,
       const share::schema::ObTableSchema &orig_table_schema,
+      const uint64_t tenant_data_version,
       share::schema::ObTableSchema &new_table_schema,
       common::ObIArray<share::schema::ObColumnSchemaV2*> &new_columns,
       share::schema::ObTableSchema &index_schema);
@@ -248,6 +255,7 @@ public:
                                       ObDDLSQLTransaction &trans,
                                       common::ObIAllocator &allocator,
                                       share::schema::ObSchemaGetterGuard &schema_guard,
+                                      const uint64_t tenant_data_version,
                                       ObDDLTaskRecord &task_record);
   /**
    * For recover restore table ddl task, it is a cross-tenant task, including,
@@ -300,7 +308,8 @@ public:
   int create_index_tablet(const ObTableSchema &index_schema,
                           ObMySQLTransaction &trans,
                           share::schema::ObSchemaGetterGuard &schema_guard,
-                          const bool need_check_tablet_cnt);
+                          const bool need_check_tablet_cnt,
+                          const uint64_t tenant_data_version);
   virtual int alter_table_index(const obrpc::ObAlterTableArg &alter_table_arg,
                                 const share::schema::ObTableSchema &orgin_table_schema,
                                 share::schema::ObTableSchema &new_table_schema,
@@ -308,6 +317,7 @@ public:
                                 ObDDLOperator &ddl_operator,
                                 ObMySQLTransaction &trans,
                                 common::ObArenaAllocator &allocator,
+                                const uint64_t tenant_data_version,
                                 obrpc::ObAlterTableRes &res,
                                 ObIArray<ObDDLTaskRecord> &ddl_tasks);
   int generate_object_id_for_partition_schemas(
@@ -344,7 +354,7 @@ public:
       share::schema::ObSchemaGetterGuard &schema_guard,
       ObDDLOperator *ddl_operator,
       common::ObMySQLTransaction *trans);
-  int add_column_group_to_table_schema(
+  int add_column_to_column_group(
       const share::schema::ObTableSchema &origin_table_schema,
       const share::schema::AlterTableSchema &alter_table_schema,
       share::schema::ObTableSchema &new_table_schema,
@@ -697,7 +707,8 @@ int check_table_udt_id_is_exist(share::schema::ObSchemaGetterGuard &schema_guard
       ObDDLOperator &ddl_operator,
       common::ObMySQLTransaction &trans,
       common::ObArenaAllocator &allocator,
-      const obrpc::ObIndexArg::IndexActionType &index_action_type);
+      const obrpc::ObIndexArg::IndexActionType &index_action_type,
+      const uint64_t tenant_data_version);
   int get_add_pk_index_name(const share::schema::ObTableSchema &origin_table_schema,
                             share::schema::ObTableSchema &new_table_schema,
                             const obrpc::ObIndexArg::IndexActionType &index_action_type,
@@ -1246,7 +1257,8 @@ private:
       const uint64_t tenant_id,
       const share::schema::ObTableSchema &orig_table_schema,
       ObDDLOperator &ddl_operator,
-      ObMySQLTransaction &trans);
+      ObMySQLTransaction &trans,
+      const uint64_t tenant_data_version);
   int fill_interval_info_for_set_interval(const ObTableSchema &orig_table_schema,
       ObTableSchema &new_table_schema,
       AlterTableSchema &inc_table_schema);
@@ -1287,15 +1299,18 @@ private:
                             const common::ObString *ddl_stmt_str,
                             ObMySQLTransaction *sql_trans,
                             share::schema::ObSchemaGetterGuard &schema_guard,
-                            const bool need_check_tablet_cnt);
+                            const bool need_check_tablet_cnt,
+                            const uint64_t tenant_data_version);
   int create_tablets_in_trans_(common::ObIArray<share::schema::ObTableSchema> &table_schemas,
                               ObDDLOperator &ddl_operator,
                               ObMySQLTransaction &trans,
-                              share::schema::ObSchemaGetterGuard &schema_guard);
+                              share::schema::ObSchemaGetterGuard &schema_guard,
+                              const uint64_t tenant_data_version);
   int create_tablets_in_trans_for_mv_(common::ObIArray<share::schema::ObTableSchema> &table_schemas,
                               ObDDLOperator &ddl_operator,
                               ObMySQLTransaction &trans,
-                              share::schema::ObSchemaGetterGuard &schema_guard);
+                              share::schema::ObSchemaGetterGuard &schema_guard,
+                              const uint64_t tenant_data_version);
 
   /*
    * Check and set various options of modify tenant, among which the modifications of zone_list,
@@ -1452,6 +1467,7 @@ private:
       ObDDLOperator &ddl_operator,
       common::ObMySQLTransaction &trans,
       common::ObIAllocator &allocator,
+      const uint64_t tenant_data_version,
       const ObString &index_name = ObString(""));
   int rebuild_triggers_on_hidden_table(
       const share::schema::ObTableSchema &orig_table_schema,
@@ -1557,7 +1573,8 @@ private:
                               share::schema::ObSchemaGetterGuard &schema_guard,
                               ObDDLOperator &ddl_operator,
                               common::ObMySQLTransaction &trans,
-                              common::ObArenaAllocator &allocator);
+                              common::ObArenaAllocator &allocator,
+                              const uint64_t tenant_data_version);
   int check_alter_partition_with_tablegroup(const ObTableSchema *orig_table_schema,
                                             ObTableSchema &new_table_schema,
                                             ObSchemaGetterGuard &schema_guard);
@@ -1566,7 +1583,8 @@ private:
                               share::schema::ObTableSchema &new_table_schema,
                               share::schema::ObSchemaGetterGuard &schema_guard,
                               ObDDLOperator &ddl_operator,
-                              common::ObMySQLTransaction &trans);
+                              common::ObMySQLTransaction &trans,
+                              const uint64_t tenant_data_version);
   int convert_to_character_for_partition(const ObCollationType &to_collation,
                            share::schema::ObTableSchema &new_table_schema);
   int convert_to_character(obrpc::ObAlterTableArg &alter_table_arg,
@@ -1574,7 +1592,17 @@ private:
                            share::schema::ObTableSchema &new_table_schema,
                            share::schema::ObSchemaGetterGuard &schema_guard,
                            ObDDLOperator &ddl_operator,
-                           common::ObMySQLTransaction &trans);
+                           common::ObMySQLTransaction &trans,
+                           const uint64_t tenant_data_version);
+  int check_alter_column_group(const obrpc::ObAlterTableArg &alter_table_arg, share::ObDDLType &ddl_type) const;
+  int alter_column_group(obrpc::ObAlterTableArg &alter_table_arg,
+                         const share::schema::ObTableSchema &origin_table_schema,
+                         share::schema::ObTableSchema &new_table_schema,
+                         share::schema::ObSchemaGetterGuard &schema_guard,
+                         ObDDLOperator &ddl_operator,
+                         common::ObMySQLTransaction &trans);
+
+
   int check_alter_table_constraint(
       const obrpc::ObAlterTableArg &alter_table_arg,
       const ObTableSchema &orig_table_schema,
@@ -1888,6 +1916,11 @@ private:
       ObDDLOperator *ddl_operator,
       common::ObMySQLTransaction *trans);
 
+  int alter_table_update_cg_column(common::ObMySQLTransaction &trans,
+                                   ObDDLOperator &ddl_operator,
+                                   share::schema::ObColumnSchemaV2 &new_column_schema,
+                                   share::schema::ObTableSchema &new_table_schema);
+
   bool is_zone_exist(const common::ObArray<common::ObZone> &zones, const common::ObZone &zone);
   int try_drop_sys_ls_(const uint64_t meta_tenant_id,
                        common::ObMySQLTransaction &trans);
@@ -2035,6 +2068,16 @@ public:
              common::ObIAllocator *allocator = NULL);
 #endif
 private:
+  int adjust_cg_for_offline(ObTableSchema &new_table_schema);
+  int alter_default_column_group(share::schema::ObTableSchema &new_table_schema);
+  int add_column_group(const obrpc::ObAlterTableArg &alter_table_arg,
+                       const share::schema::ObTableSchema &ori_table_schema,
+                       share::schema::ObTableSchema &new_table_schema);
+
+  int drop_column_group(const obrpc::ObAlterTableArg &alter_table_arg,
+                        const share::schema::ObTableSchema &ori_table_schema,
+                        share::schema::ObTableSchema &new_table_schema);
+  int alter_rowkey_column_group(share::schema::ObTableSchema &table_schema);
   int handle_security_audit_for_stmt(const obrpc::ObSecurityAuditArg &arg,
                                      share::schema::ObSAuditSchema &audit_schema);
   int handle_security_audit_for_object(const obrpc::ObSecurityAuditArg &arg,
@@ -2656,6 +2699,7 @@ private:
                                         const ObTableSchema &container_table_schema,
                                         const ObIArray<ObDependencyInfo> *dep_infos,
                                         common::ObIAllocator &allocator,
+                                        const uint64_t tenant_data_version,
                                         ObDDLTaskRecord &task_record);
 
   bool need_modify_dep_obj_status(const obrpc::ObAlterTableArg &alter_table_arg) const;

src/rootserver/ob_index_builder.cpp

@@ -294,6 +294,7 @@ int ObIndexBuilder::do_create_global_index(
     obrpc::ObAlterTableRes &res)
 {
   int ret = OB_SUCCESS;
+  uint64_t tenant_data_version = 0;
   ObArray<ObColumnSchemaV2 *> gen_columns;
   const bool global_index_without_column_info = false;
   ObDDLTaskRecord task_record;
@@ -325,15 +326,17 @@ int ObIndexBuilder::do_create_global_index(
         new_arg, new_table_schema, global_index_without_column_info,
         true/*generate_id*/, index_schema))) {
       LOG_WARN("fail to generate schema", K(ret), K(new_arg));
+    } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
+      LOG_WARN("get min data version failed", K(ret), K(tenant_id));
     } else {
       if (gen_columns.empty()) {
         if (OB_FAIL(ddl_service_.create_global_index(
-                trans, new_arg, new_table_schema, index_schema))) {
+                trans, new_arg, new_table_schema, tenant_data_version, index_schema))) {
           LOG_WARN("fail to create global index", K(ret));
         }
       } else {
         if (OB_FAIL(ddl_service_.create_global_inner_expr_index(
-              trans, table_schema, new_table_schema, gen_columns, index_schema))) {
+              trans, table_schema, tenant_data_version, new_table_schema, gen_columns, index_schema))) {
           LOG_WARN("fail to create global inner expr index", K(ret));
         }
       }
@@ -345,9 +348,10 @@ int ObIndexBuilder::do_create_global_index(
                                                  nullptr/*del_data_tablet_ids*/,
                                                  &index_schema,
                                                  arg.parallelism_,
+                                                 arg.consumer_group_id_,
+                                                 tenant_data_version,
                                                  allocator,
-                                                 task_record,
-                                                 arg.consumer_group_id_))) {
+                                                 task_record))) {
         LOG_WARN("fail to submit build global index task", K(ret));
       }
     }
@@ -383,9 +387,10 @@ int ObIndexBuilder::submit_build_index_task(
     const ObIArray<ObTabletID> *del_data_tablet_ids,
     const ObTableSchema *index_schema,
     const int64_t parallelism,
+    const int64_t group_id,
+    const uint64_t tenant_data_version,
     common::ObIAllocator &allocator,
-    ObDDLTaskRecord &task_record,
-    const int64_t group_id)
+    ObDDLTaskRecord &task_record)
 {
   int ret = OB_SUCCESS;
   ObCreateDDLTaskParam param(index_schema->get_tenant_id(),
@@ -398,9 +403,10 @@ int ObIndexBuilder::submit_build_index_task(
                              group_id,
                              &allocator,
                              &create_index_arg);
-  if (OB_ISNULL(data_schema) || OB_ISNULL(index_schema)) {
+  param.tenant_data_version_ = tenant_data_version;
+  if (OB_UNLIKELY(nullptr == data_schema || nullptr == index_schema || tenant_data_version <= 0)) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("schema is invalid", K(ret), K(data_schema), K(index_schema));
+    LOG_WARN("schema is invalid", K(ret), KP(data_schema), KP(index_schema), K(tenant_data_version));
   } else if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, trans, task_record))) {
     LOG_WARN("submit create index ddl task failed", K(ret));
   } else if (OB_FAIL(ObDDLLock::lock_for_add_drop_index(
@@ -509,12 +515,13 @@ int ObIndexBuilder::do_create_local_index(
       } else if (OB_FAIL(new_table_schema.check_create_index_on_hidden_primary_key(index_schema))) {
         LOG_WARN("failed to check create index on table", K(ret), K(index_schema));
       } else if (gen_columns.empty()) {
-        if (OB_FAIL(ddl_service_.create_index_table(my_arg, index_schema, trans))) {
+        if (OB_FAIL(ddl_service_.create_index_table(my_arg, tenant_data_version, index_schema, trans))) {
           LOG_WARN("fail to create index", K(ret), K(index_schema));
         }
       } else {
         if (OB_FAIL(ddl_service_.create_inner_expr_index(trans,
                                                          table_schema,
+                                                         tenant_data_version,
                                                          new_table_schema,
                                                          gen_columns,
                                                          index_schema))) {
@@ -529,9 +536,10 @@ int ObIndexBuilder::do_create_local_index(
                                                  nullptr/*del_data_tablet_ids*/,
                                                  &index_schema,
                                                  create_index_arg.parallelism_,
+                                                 create_index_arg.consumer_group_id_,
+                                                 tenant_data_version,
                                                  allocator,
-                                                 task_record,
-                                                 create_index_arg.consumer_group_id_))) {
+                                                 task_record))) {
         LOG_WARN("failt to submit build local index task", K(ret));
       } else {
         res.index_table_id_ = index_schema.get_table_id();
@@ -825,8 +833,6 @@ int ObIndexBuilder::generate_schema(
         LOG_WARN("set_index_table_columns failed", K(arg), K(data_schema), K(ret));
       } else if (OB_FAIL(set_index_table_options(arg, data_schema, schema))) {
         LOG_WARN("set_index_table_options failed", K(arg), K(data_schema), K(ret));
-      } else if (OB_FAIL(set_index_table_column_store_if_need(schema))) {
-        LOG_WARN("fail to set index table column store if need", KR(ret));
       } else {
         schema.set_name_generated_type(arg.index_schema_.get_name_generated_type());
         LOG_INFO("finish generate index schema", K(schema));
@@ -847,6 +853,121 @@ int ObIndexBuilder::generate_schema(
       }
     }
   }
+
+  if (OB_SUCC(ret)) {
+    // create index column_group after schema generate
+    if (OB_FAIL(create_index_column_group(arg, schema))) {
+      LOG_WARN("fail to create cg for index", K(ret));
+    }
+  }
+  return ret;
+}
+
+int ObIndexBuilder::create_index_column_group(const obrpc::ObCreateIndexArg &arg, ObTableSchema &index_table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t compat_version = 0;
+  if (OB_FAIL(GET_MIN_DATA_VERSION(index_table_schema.get_tenant_id(), compat_version))) {
+    LOG_WARN("fail to get min data version", K(ret));
+  } else if (compat_version >= DATA_VERSION_4_3_0_0) {
+    bool enable_table_with_cg = false;
+    ObArray<uint64_t> column_ids; // not include virtual column
+    index_table_schema.set_column_store(true);
+    if (arg.index_cgs_.count() > 0) {
+      index_table_schema.set_max_used_column_group_id(index_table_schema.get_max_used_column_group_id());
+      for (int64_t i = 0; OB_SUCC(ret) && i < arg.index_cgs_.count(); ++i) {
+        const obrpc::ObCreateIndexArg::ObIndexColumnGroupItem &cur_item = arg.index_cgs_.at(i);
+        if (!cur_item.is_valid()) {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_WARN("invalid cg item", K(ret), K(cur_item));
+        } else if (cur_item.is_each_cg_) {
+          // handle all_type column_group & single_type column_group
+          ObColumnGroupSchema column_group_schema;
+          const int64_t column_cnt = index_table_schema.get_column_count();
+          if (OB_FAIL(column_ids.reserve(column_cnt))) {
+            LOG_WARN("fail to reserve", KR(ret), K(column_cnt));
+          } else {
+            ObTableSchema::const_column_iterator tmp_begin = index_table_schema.column_begin();
+            ObTableSchema::const_column_iterator tmp_end = index_table_schema.column_end();
+            for (; OB_SUCC(ret) && (tmp_begin != tmp_end); tmp_begin++) {
+              column_group_schema.reset();
+              ObColumnSchemaV2 *column = (*tmp_begin);
+              if (OB_FAIL(ObSchemaUtils::build_single_column_group(
+                                          index_table_schema, column, index_table_schema.get_tenant_id(),
+                                          index_table_schema.get_max_used_column_group_id() + 1, column_group_schema))) {
+                LOG_WARN("fail to build single column group");
+              } else if (column_group_schema.is_valid()) {
+                if (OB_FAIL(index_table_schema.add_column_group(column_group_schema))) {
+                  LOG_WARN("fail to add single type column group", KR(ret), K(column_group_schema));
+                } else if (column->is_rowkey_column() || arg.exist_all_column_group_) {//if not exist all cg, build rowkey cg
+                  if (OB_FAIL(column_ids.push_back(column->get_column_id()))) {
+                    LOG_WARN("fail to push back", KR(ret), "column_id", column->get_column_id());
+                  }
+                }
+              }
+            }
+          }
+
+          if (OB_SUCC(ret)) {
+            column_group_schema.reset();
+            const ObColumnGroupType cg_type = arg.exist_all_column_group_ ? ObColumnGroupType::ALL_COLUMN_GROUP
+                                              : ObColumnGroupType::ROWKEY_COLUMN_GROUP;
+            const ObString cg_name = arg.exist_all_column_group_ ? OB_ALL_COLUMN_GROUP_NAME : OB_ROWKEY_COLUMN_GROUP_NAME;
+
+            if (OB_FAIL(ObSchemaUtils::build_column_group(index_table_schema, index_table_schema.get_tenant_id(), cg_type, cg_name,
+                column_ids, index_table_schema.get_max_used_column_group_id() + 1, column_group_schema))) {
+              LOG_WARN("fail to build all type column_group", KR(ret), K(column_ids));
+            } else if (OB_FAIL(index_table_schema.add_column_group(column_group_schema))) {
+              LOG_WARN("fail to add all type column group", KR(ret), K(column_group_schema));
+            }
+          }
+        }
+      }
+    } else {
+      omt::ObTenantConfigGuard tenant_config(TENANT_CONF(index_table_schema.get_tenant_id()));
+      if (OB_SUCC(ret) && OB_LIKELY(tenant_config.is_valid())) {
+        if (tenant_config->enable_table_with_cg) {
+          enable_table_with_cg = true; // which means create each_cg and all_cg default
+        }
+      }
+    }
+
+    // add default column_group
+    if (OB_SUCC(ret)) {
+      ObColumnGroupSchema tmp_cg;
+      if (arg.index_cgs_.count() > 0 || enable_table_with_cg) {
+        column_ids.reuse(); // if exists cg node, column_ids in default_type will be empty
+      } else {
+        ObTableSchema::const_column_iterator tmp_begin = index_table_schema.column_begin();
+        ObTableSchema::const_column_iterator tmp_end = index_table_schema.column_end();
+        for (; OB_SUCC(ret) && (tmp_begin != tmp_end); tmp_begin++) {
+          ObColumnSchemaV2 *column = (*tmp_begin);
+          if (OB_ISNULL(column)) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("column should not be null", KR(ret));
+          } else if (column->is_virtual_generated_column()) {
+            // skip virtual column
+          } else if (OB_FAIL(column_ids.push_back(column->get_column_id()))) {
+            LOG_WARN("fail to push back", KR(ret), "column_id", column->get_column_id());
+          }
+        }
+      }
+
+      if (FAILEDx(ObSchemaUtils::build_column_group(index_table_schema, index_table_schema.get_tenant_id(),
+          ObColumnGroupType::DEFAULT_COLUMN_GROUP, OB_DEFAULT_COLUMN_GROUP_NAME, column_ids,
+          DEFAULT_TYPE_COLUMN_GROUP_ID, tmp_cg))) {
+        LOG_WARN("fail to build default type column_group", KR(ret), "table_id", index_table_schema.get_table_id(), K(column_ids));
+      } else if (OB_FAIL(index_table_schema.add_column_group(tmp_cg))) {
+        LOG_WARN("fail to add default column group", KR(ret), "table_id", index_table_schema.get_table_id(), K(arg.index_cgs_.count()),
+                                                     K(enable_table_with_cg), K(column_ids));
+      }
+    }
+  } else if (arg.index_cgs_.count() > 0) {
+    ret = OB_NOT_SUPPORTED;
+    LOG_WARN("data_version not support for create index with column group", K(ret), K(compat_version));
+    LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3, create index with column group");
+  }
+
   return ret;
 }
 
@@ -1000,28 +1121,5 @@ bool ObIndexBuilder::is_final_index_status(const ObIndexStatus index_status) con
           || is_error_index_status(index_status));
 }
 
-int ObIndexBuilder::set_index_table_column_store_if_need(
-    share::schema::ObTableSchema &table_schema)
-{
-  int ret = OB_SUCCESS;
-  uint64_t compat_version = 0;
-  const uint64_t tenant_id = table_schema.get_tenant_id();
-  const uint64_t table_id = table_schema.get_table_id();
-  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(table_schema));
-  } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
-    LOG_WARN("fail to get min data version", KR(ret), K(tenant_id), K(table_id));
-  } else if (compat_version >= DATA_VERSION_4_2_0_0) {
-    table_schema.set_column_store(true);
-    if (table_schema.get_column_group_count() == 0) {
-      if (OB_FAIL(table_schema.add_default_column_group())) {
-        LOG_WARN("fail to add default column group", KR(ret), K(tenant_id), K(table_id));
-      }
-    }
-  }
-  return ret;
-}
-
 }//end namespace rootserver
 }//end namespace oceanbase

src/rootserver/ob_index_builder.h

@@ -93,9 +93,10 @@ public:
                               const common::ObIArray<common::ObTabletID> *del_data_tablet_ids,
                               const share::schema::ObTableSchema *index_schema,
                               const int64_t parallelism,
+                              const int64_t group_id,
+                              const uint64_t tenant_data_version,
                               common::ObIAllocator &allocator,
-                              ObDDLTaskRecord &task_record,
-                              const int64_t group_id);
+                              ObDDLTaskRecord &task_record);
 private:
   typedef common::ObArray<std::pair<int64_t, common::ObString> > OrderFTColumns;
   class FulltextColumnOrder
@@ -123,7 +124,8 @@ private:
 
   bool is_final_index_status(const share::schema::ObIndexStatus index_status) const;
 
-  int set_index_table_column_store_if_need(share::schema::ObTableSchema &table_schema);
+  int create_index_column_group(const obrpc::ObCreateIndexArg &arg,
+                                share::schema::ObTableSchema &index_table_schema);
 
 private:
   ObDDLService &ddl_service_;

src/rootserver/ob_lob_meta_builder.cpp

@@ -183,7 +183,7 @@ int ObLobMetaBuilder::set_lob_table_column_store_if_need(ObTableSchema &table_sc
   const uint64_t tenant_id = table_schema.get_tenant_id();
   if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
     LOG_WARN("fail to get min data version", KR(ret), K(tenant_id), "table_id", table_schema.get_table_id());
-  } else if (compat_version >= DATA_VERSION_4_2_0_0) {
+  } else if (compat_version >= DATA_VERSION_4_3_0_0) {
     table_schema.set_column_store(true);
     if (table_schema.get_column_group_count() == 0) {
       if (OB_FAIL(table_schema.add_default_column_group())) {

src/rootserver/ob_lob_piece_builder.cpp

@@ -183,7 +183,7 @@ int ObLobPieceBuilder::set_lob_table_column_store_if_need(ObTableSchema &table_s
   const uint64_t tenant_id = table_schema.get_tenant_id();
   if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
     LOG_WARN("fail to get min data version", KR(ret), K(tenant_id), "table_id", table_schema.get_table_id());
-  } else if (compat_version >= DATA_VERSION_4_2_0_0) {
+  } else if (compat_version >= DATA_VERSION_4_3_0_0) {
     table_schema.set_column_store(true);
     if (table_schema.get_column_group_count() == 0) {
       if (OB_FAIL(table_schema.add_default_column_group())) {

src/rootserver/ob_mlog_builder.cpp

@@ -396,6 +396,7 @@ int ObMLogBuilder::create_mlog(
     } else if (OB_FAIL(do_create_mlog(schema_guard,
                                       create_mlog_arg,
                                       *base_table_schema,
+                                      compat_version,
                                       create_mlog_res))) {
       LOG_WARN("failed to do create mlog", KR(ret), K(create_mlog_arg));
     }
@@ -407,6 +408,7 @@ int ObMLogBuilder::do_create_mlog(
     ObSchemaGetterGuard &schema_guard,
     const ObCreateMLogArg &create_mlog_arg,
     const ObTableSchema &base_table_schema,
+    const uint64_t tenant_data_version,
     ObCreateMLogRes &create_mlog_res)
 {
   int ret = OB_SUCCESS;
@@ -431,7 +433,7 @@ int ObMLogBuilder::do_create_mlog(
       LOG_WARN("failed to copy table schema", KR(ret));
     } else if (OB_FAIL(generate_mlog_schema(schema_guard, create_mlog_arg, src_table_schema, mlog_schema))) {
       LOG_WARN("failed to generate schema", KR(ret), K(create_mlog_arg), K(src_table_schema));
-    } else if (OB_FAIL(ddl_service_.create_mlog_table(trans, create_mlog_arg, schema_guard, mlog_schema))) {
+    } else if (OB_FAIL(ddl_service_.create_mlog_table(trans, create_mlog_arg, tenant_data_version, schema_guard, mlog_schema))) {
       LOG_WARN("failed to create mlog table", KR(ret), K(create_mlog_arg), K(mlog_schema));
     } else {
       // submit build mlog task
@@ -451,6 +453,7 @@ int ObMLogBuilder::do_create_mlog(
                                   create_mlog_arg.consumer_group_id_,
                                   &allocator,
                                   &create_index_arg);
+      param.tenant_data_version_ = tenant_data_version;
       if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, trans, task_record))) {
         LOG_WARN("failed to submit create mlog task", KR(ret));
       } else if (OB_FAIL(ObDDLLock::lock_for_add_drop_index(

src/rootserver/ob_mlog_builder.h

@@ -33,6 +33,7 @@ public:
   int do_create_mlog(share::schema::ObSchemaGetterGuard &schema_guard,
                      const obrpc::ObCreateMLogArg &create_mlog_arg,
                      const share::schema::ObTableSchema &table_schema,
+                     const uint64_t tenant_data_version,
                      obrpc::ObCreateMLogRes &create_mlog_res);
   int generate_mlog_schema(share::schema::ObSchemaGetterGuard &schema_guard,
                            const obrpc::ObCreateMLogArg &create_mlog_arg,
@@ -86,4 +87,4 @@ private:
 };
 } // namespace rootserver
 } // namespace oceanbase
-#endif  // OCEANBASE_ROOTSERVER_OB_MLOG_BUILDER_H_
+#endif  // OCEANBASE_ROOTSERVER_OB_MLOG_BUILDER_H_

src/rootserver/ob_schema_history_recycler.cpp

@@ -796,6 +796,9 @@ int ObSchemaHistoryRecycler::try_recycle_schema_history(
     RECYCLE_FIRST_SCHEMA(RECYCLE_ONLY, tablet, OB_ALL_TABLET_TO_TABLE_HISTORY_TNAME, tablet_id);
     ret = OB_SUCCESS; // overwrite ret
 
+    RECYCLE_SECOND_SCHEMA(column_group, OB_ALL_COLUMN_GROUP_HISTORY_TNAME, table_id, column_group_id);
+    RECYCLE_SECOND_SCHEMA(column_group_mapping, OB_ALL_COLUMN_GROUP_MAPPING_HISTORY_TNAME, table_id, column_group_id);
+    ret = OB_SUCCESS; // overwrite ret
     // ----------------------------- database ----------------------------------------
     RECYCLE_FIRST_SCHEMA(RECYCLE_AND_COMPRESS, database, OB_ALL_DATABASE_HISTORY_TNAME,
                          database_id);

src/rootserver/ob_table_creator.cpp

@@ -59,7 +59,9 @@ int ObTableCreator::execute()
 int ObTableCreator::add_create_tablets_of_local_aux_tables_arg(
                     const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
                     const share::schema::ObTableSchema *data_table_schema,
-                    const common::ObIArray<share::ObLSID> &ls_id_array)
+                    const common::ObIArray<share::ObLSID> &ls_id_array,
+                    const uint64_t tenant_data_version,
+                    const common::ObIArray<bool> &need_create_empty_majors)
 {
   int ret = OB_SUCCESS;
   if (OB_ISNULL(data_table_schema)) {
@@ -71,6 +73,10 @@ int ObTableCreator::add_create_tablets_of_local_aux_tables_arg(
       data_table_schema->is_mlog_table()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("data_table_schema must be data table", KR(ret), KPC(data_table_schema));
+  } else if (OB_UNLIKELY(tenant_data_version <= 0 || need_create_empty_majors.count() != schemas.count())) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid args", K(ret), K(tenant_data_version), "count_need_create_empty_majors", need_create_empty_majors.count(),
+      "count_schemas", schemas.count());
   }
   for (int64_t i = 0; OB_SUCC(ret) && i < schemas.count(); ++i) {
     const share::schema::ObTableSchema *aux_schema = schemas.at(i);
@@ -86,7 +92,7 @@ int ObTableCreator::add_create_tablets_of_local_aux_tables_arg(
   }
   if (OB_FAIL(ret)) {
   } else if (OB_FAIL(add_create_tablets_of_tables_arg_(
-          schemas, data_table_schema, ls_id_array))) {
+          schemas, data_table_schema, ls_id_array, tenant_data_version, need_create_empty_majors))) {
     LOG_WARN("fail to add_create_tablets_of_tables_arg_", KR(ret), K(schemas));
   }
   return ret;
@@ -95,10 +101,12 @@ int ObTableCreator::add_create_tablets_of_local_aux_tables_arg(
 int ObTableCreator::add_create_bind_tablets_of_hidden_table_arg(
                     const share::schema::ObTableSchema &orig_table_schema,
                     const share::schema::ObTableSchema &hidden_table_schema,
-                    const common::ObIArray<share::ObLSID> &ls_id_array)
+                    const common::ObIArray<share::ObLSID> &ls_id_array,
+                    const uint64_t tenant_data_version)
 {
   int ret = OB_SUCCESS;
   ObSEArray<const ObTableSchema *, 1> schemas;
+  ObSEArray<bool, 1> need_create_empty_majors;
   if (OB_UNLIKELY(!orig_table_schema.has_tablet()
       || orig_table_schema.is_index_table()
       || hidden_table_schema.is_index_table()
@@ -107,10 +115,10 @@ int ObTableCreator::add_create_bind_tablets_of_hidden_table_arg(
       || !hidden_table_schema.is_user_hidden_table())) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("both orig and hidden table must be data table", K(ret), K(orig_table_schema), K(hidden_table_schema));
-  } else if (OB_FAIL(schemas.push_back(&hidden_table_schema))) {
+  } else if (OB_FAIL(schemas.push_back(&hidden_table_schema)) || OB_FAIL(need_create_empty_majors.push_back(false))) {
     LOG_WARN("failed to push back hidden table schema", K(ret));
   } else if (OB_FAIL(add_create_tablets_of_tables_arg_(
-          schemas, &orig_table_schema, ls_id_array))) {
+          schemas, &orig_table_schema, ls_id_array, tenant_data_version, need_create_empty_majors))) {
     LOG_WARN("failed to add arg", K(ret), K(schemas));
   }
   return ret;
@@ -118,20 +126,21 @@ int ObTableCreator::add_create_bind_tablets_of_hidden_table_arg(
 
 int ObTableCreator::add_create_tablets_of_table_arg(
                     const share::schema::ObTableSchema &table_schema,
-                    const common::ObIArray<share::ObLSID> &ls_id_array)
+                    const common::ObIArray<share::ObLSID> &ls_id_array,
+                    const uint64_t tenant_data_version,
+                    const bool need_create_empty_major_sstable)
 {
   int ret = OB_SUCCESS;
   ObSEArray<const share::schema::ObTableSchema*, 1> schemas;
-  if (!table_schema.has_tablet()
-      || table_schema.is_index_local_storage()
-      || table_schema.is_aux_lob_table()
-      || table_schema.is_mlog_table()) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("table_schema must be data table or global indexes", KR(ret), K(table_schema));
-  } else if (OB_FAIL(schemas.push_back(&table_schema))) {
-    LOG_WARN("failed to push_back", KR(ret), K(table_schema));
+  ObSEArray<bool, 1> need_create_empty_majors;
+  if (!table_schema.has_tablet() || table_schema.is_index_local_storage() || table_schema.is_aux_lob_table()
+    || table_schema.is_mlog_table() || tenant_data_version <= 0) {
+    LOG_WARN("table_schema must be data table or global indexes", KR(ret), K(table_schema), K(tenant_data_version));
+  } else if (OB_FAIL(schemas.push_back(&table_schema))
+    || OB_FAIL(need_create_empty_majors.push_back(need_create_empty_major_sstable))) {
+    LOG_WARN("failed to push_back", KR(ret), K(table_schema), K(need_create_empty_major_sstable));
   } else if (OB_FAIL(add_create_tablets_of_tables_arg_(
-          schemas, NULL, ls_id_array))) {
+          schemas, NULL, ls_id_array, tenant_data_version, need_create_empty_majors))) {
     LOG_WARN("failed to add create tablet arg", KR(ret), K(table_schema));
   }
   return ret;
@@ -139,9 +148,17 @@ int ObTableCreator::add_create_tablets_of_table_arg(
 
 int ObTableCreator::add_create_tablets_of_tables_arg(
                     const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
-                    const common::ObIArray<share::ObLSID> &ls_id_array)
+                    const common::ObIArray<share::ObLSID> &ls_id_array,
+                    const uint64_t tenant_data_version,
+                    const common::ObIArray<bool> &need_create_empty_majors)
 {
   int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(tenant_data_version <= 0
+    || schemas.count() != need_create_empty_majors.count())) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid args", K(ret), K(tenant_data_version), "count_schemas", schemas.count(),
+      "count_need_create_empty_majors", need_create_empty_majors.count());
+  }
   for (int64_t i = 0; OB_SUCC(ret) && i < schemas.count(); ++i) {
     const share::schema::ObTableSchema *table_schema = schemas.at(i);
     if (OB_ISNULL(table_schema)) {
@@ -166,7 +183,7 @@ int ObTableCreator::add_create_tablets_of_tables_arg(
   }
   if (OB_FAIL(ret)) {
   } else if (OB_FAIL(add_create_tablets_of_tables_arg_(
-          schemas, NULL, ls_id_array))) {
+          schemas, NULL, ls_id_array, tenant_data_version, need_create_empty_majors))) {
     LOG_WARN("fail to add_create_tablets_of_tables_arg_", KR(ret), K(schemas));
   }
   return ret;
@@ -178,13 +195,17 @@ int ObTableCreator::add_create_tablets_of_tables_arg(
 int ObTableCreator::add_create_tablets_of_tables_arg_(
                     const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
                     const share::schema::ObTableSchema *data_table_schema,
-                    const common::ObIArray<share::ObLSID> &ls_id_array)
+                    const common::ObIArray<share::ObLSID> &ls_id_array,
+                    const uint64_t tenant_data_version,
+                    const common::ObIArray<bool> &need_create_empty_majors)
 {
   int ret = OB_SUCCESS;
   const int64_t schema_cnt = schemas.count();
-  if (OB_UNLIKELY(schema_cnt < 1)) {
+  if (OB_UNLIKELY(schema_cnt < 1 || tenant_data_version <= 0
+    || schema_cnt != need_create_empty_majors.count())) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("schemas count is less 1", KR(ret), K(schema_cnt));
+    LOG_WARN("schemas count is less 1", KR(ret), K(schema_cnt), K(tenant_data_version),
+      "create_major_flag_cnt", need_create_empty_majors.count());
   } else if (OB_ISNULL(schemas.at(0))) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("NULL ptr", KR(ret), K(schemas));
@@ -268,7 +289,9 @@ int ObTableCreator::add_create_tablets_of_tables_arg_(
                     pairs,
                     OB_INVALID_INDEX,
                     OB_INVALID_INDEX,
-                    is_create_bind_hidden_tablets))) {
+                    is_create_bind_hidden_tablets,
+                    tenant_data_version,
+                    need_create_empty_majors))) {
           LOG_WARN("fail to generate_create_tablet_arg",
                    K(table_schema), K(schemas), KR(ret), K(is_create_bind_hidden_tablets));
         }
@@ -292,7 +315,9 @@ int ObTableCreator::add_create_tablets_of_tables_arg_(
                           pairs,
                           i,
                           OB_INVALID_INDEX,
-                          is_create_bind_hidden_tablets))) {
+                          is_create_bind_hidden_tablets,
+                          tenant_data_version,
+                          need_create_empty_majors))) {
                 LOG_WARN("fail to generate_create_tablet_arg",
                          K(table_schema), K(schemas), KR(ret), K(i), K(is_create_bind_hidden_tablets));
               }
@@ -316,7 +341,9 @@ int ObTableCreator::add_create_tablets_of_tables_arg_(
                                 pairs,
                                 i,
                                 j,
-                                is_create_bind_hidden_tablets))) {
+                                is_create_bind_hidden_tablets,
+                                tenant_data_version,
+                                need_create_empty_majors))) {
                       LOG_WARN("fail to generate_create_tablet_arg",
                                K(table_schema), K(schemas), KR(ret), K(i), K(j), K(is_create_bind_hidden_tablets));
                     }
@@ -355,7 +382,9 @@ int ObTableCreator::generate_create_tablet_arg_(
                     common::ObIArray<share::ObTabletTablePair> &pairs,
                     const int64_t part_idx,
                     const int64_t subpart_idx,
-                    const bool is_create_bind_hidden_tablets)
+                    const bool is_create_bind_hidden_tablets,
+                    const uint64_t tenant_data_version,
+                    const common::ObIArray<bool> &need_create_empty_majors)
 {
   int ret = OB_SUCCESS;
   ObTabletID data_tablet_id;
@@ -416,7 +445,9 @@ int ObTableCreator::generate_create_tablet_arg_(
                      data_tablet_id,
                      schemas,
                      mode,
-                     is_create_bind_hidden_tablets))) {
+                     is_create_bind_hidden_tablets,
+                     tenant_data_version,
+                     need_create_empty_majors))) {
     LOG_WARN("fail to init create tablet arg", KR(ret), K(schemas), K(is_create_bind_hidden_tablets));
   } else if (OB_FAIL(tablet_creator_.add_create_tablet_arg(create_tablet_arg))) {
     LOG_WARN("fail to add create tablet arg", KR(ret), K(create_tablet_arg));

src/rootserver/ob_table_creator.h

@@ -48,7 +48,9 @@ public:
   // @param [in] schemas, tables schema for creating tablets, the first is data table, others are its local indexes
   int add_create_tablets_of_tables_arg(
       const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
-      const common::ObIArray<share::ObLSID> &ls_id_array);
+      const common::ObIArray<share::ObLSID> &ls_id_array,
+      const uint64_t tenant_data_version,
+      const common::ObIArray<bool> &need_create_empty_majors);
 
   // create tablets for local aux tables(include local_index/aux_lob_table), which are belong to a data table.
   //
@@ -57,25 +59,32 @@ public:
   int add_create_tablets_of_local_aux_tables_arg(
       const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
       const share::schema::ObTableSchema *data_table_schema,
-      const common::ObIArray<share::ObLSID> &ls_id_array);
+      const common::ObIArray<share::ObLSID> &ls_id_array,
+      const uint64_t tenant_data_version,
+      const common::ObIArray<bool> &need_create_empty_majors);
 
   // create tablets of hidden table from original table, used by ddl table redefinition
   int add_create_bind_tablets_of_hidden_table_arg(
       const share::schema::ObTableSchema &orig_table_schema,
       const share::schema::ObTableSchema &hidden_table_schema,
-      const common::ObIArray<share::ObLSID> &ls_id_array);
+      const common::ObIArray<share::ObLSID> &ls_id_array,
+      const uint64_t tenant_data_version);
 
   // create tablets in a table
   //
   // @param [in] table_schema, table schema for creating tablets
   int add_create_tablets_of_table_arg(
       const share::schema::ObTableSchema &table_schema,
-      const common::ObIArray<share::ObLSID> &ls_id_array);
+      const common::ObIArray<share::ObLSID> &ls_id_array,
+      const uint64_t tenant_data_version,
+      const bool need_create_empty_major_sstable);
 private:
   int add_create_tablets_of_tables_arg_(
       const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
       const share::schema::ObTableSchema *data_table_schema,
-      const common::ObIArray<share::ObLSID> &ls_id_array);
+      const common::ObIArray<share::ObLSID> &ls_id_array,
+      const uint64_t tenant_data_version,
+      const common::ObIArray<bool> &need_create_empty_majors);
   int generate_create_tablet_arg_(
       const common::ObIArray<const share::schema::ObTableSchema*> &schemas,
       const ObTableSchema &data_table_schema,
@@ -84,7 +93,9 @@ private:
       common::ObIArray<share::ObTabletTablePair> &pairs,
       const int64_t part_idx,
       const int64_t subpart_idx,
-      const bool is_create_bind_hidden_tablets);
+      const bool is_create_bind_hidden_tablets,
+      const uint64_t tenant_data_version,
+      const common::ObIArray<bool> &need_create_empty_majors);
   int get_tablet_list_str_(
       const share::schema::ObTableSchema &table_schema,
       ObSqlString &tablet_list);

src/rootserver/ob_tablet_creator.cpp

@@ -32,7 +32,9 @@ bool ObTabletCreatorArg::is_valid() const
   bool is_valid = ls_key_.is_valid()
                   && table_schemas_.count() > 0
                   && table_schemas_.count() == tablet_ids_.count()
-                  && lib::Worker::CompatMode::INVALID != compat_mode_;
+                  && lib::Worker::CompatMode::INVALID != compat_mode_
+                  && tenant_data_version_ > 0
+                  && need_create_empty_majors_.count() == table_schemas_.count();
   for (int64_t i = 0; i < tablet_ids_.count() && is_valid; i++) {
     is_valid = tablet_ids_.at(i).is_valid();
   }
@@ -47,6 +49,8 @@ void ObTabletCreatorArg::reset()
   data_tablet_id_.reset();
   compat_mode_ = lib::Worker::CompatMode::INVALID;
   is_create_bind_hidden_tablets_ = false;
+  tenant_data_version_ = 0;
+  need_create_empty_majors_.reset();
 }
 
 int ObTabletCreatorArg::assign(const ObTabletCreatorArg &arg)
@@ -59,11 +63,14 @@ int ObTabletCreatorArg::assign(const ObTabletCreatorArg &arg)
     LOG_WARN("failed to assign table schemas", KR(ret), K(arg));
   } else if (OB_FAIL(tablet_ids_.assign(arg.tablet_ids_))) {
     LOG_WARN("failed to assign table schemas", KR(ret), K(arg));
+  } else if (OB_FAIL(need_create_empty_majors_.assign(arg.need_create_empty_majors_))) {
+    LOG_WARN("failed to assign need create empty majors", KR(ret), K(arg));
   } else {
     data_tablet_id_ = arg.data_tablet_id_;
     ls_key_ = arg.ls_key_;
     compat_mode_ = arg.compat_mode_;
     is_create_bind_hidden_tablets_ = arg.is_create_bind_hidden_tablets_;
+    tenant_data_version_ = arg.tenant_data_version_;
   }
   return ret;
 }
@@ -74,27 +81,35 @@ int ObTabletCreatorArg::init(
     const ObTabletID data_tablet_id,
     const ObIArray<const share::schema::ObTableSchema*> &table_schemas,
     const lib::Worker::CompatMode &mode,
-    const bool is_create_bind_hidden_tablets)
+    const bool is_create_bind_hidden_tablets,
+    const uint64_t tenant_data_version,
+    const ObIArray<bool> &need_create_empty_majors)
 {
   int ret = OB_SUCCESS;
   bool is_valid = ls_key.is_valid() && table_schemas.count() > 0
-                  && table_schemas.count() == tablet_ids.count();
+                  && table_schemas.count() == tablet_ids.count()
+                  && tenant_data_version > 0
+                  && need_create_empty_majors.count() == table_schemas.count();
   for (int64_t i = 0; i < tablet_ids.count() && is_valid; i++) {
     is_valid = tablet_ids.at(i).is_valid();
   }
   if (OB_UNLIKELY(!is_valid)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", KR(ret), K(tablet_ids),
-             "count", table_schemas.count(), K(tablet_ids), K(ls_key));
+             "count", table_schemas.count(), K(tablet_ids), K(ls_key),
+             K(tenant_data_version), "count_to_create_empty_major", need_create_empty_majors.count());
   } else if (OB_FAIL(tablet_ids_.assign(tablet_ids))) {
-    LOG_WARN("failed to assign table schemas", KR(ret), K(table_schemas));
+    LOG_WARN("failed to assign table schemas", KR(ret), K(tablet_ids));
   } else if (OB_FAIL(table_schemas_.assign(table_schemas))) {
     LOG_WARN("failed to assign table schemas", KR(ret), K(table_schemas));
+  } else if (OB_FAIL(need_create_empty_majors_.assign(need_create_empty_majors))) {
+    LOG_WARN("failed to assign need create empty majors", K(ret), K(need_create_empty_majors));
   } else {
     data_tablet_id_ = data_tablet_id;
     ls_key_ = ls_key;
     compat_mode_ = mode;
     is_create_bind_hidden_tablets_ = is_create_bind_hidden_tablets;
+    tenant_data_version_ = tenant_data_version;
   }
   return ret;
 }
@@ -102,7 +117,8 @@ int ObTabletCreatorArg::init(
 DEF_TO_STRING(ObTabletCreatorArg)
 {
   int64_t pos = 0;
-  J_KV(K_(compat_mode), K_(tablet_ids), K_(data_tablet_id), K_(ls_key), K_(table_schemas), K_(is_create_bind_hidden_tablets));
+  J_KV(K_(compat_mode), K_(tablet_ids), K_(data_tablet_id), K_(ls_key), K_(table_schemas), K_(is_create_bind_hidden_tablets),
+    K_(tenant_data_version), K_(need_create_empty_majors));
   return pos;
 }
 
@@ -139,12 +155,15 @@ int ObBatchCreateTabletHelper::add_arg_to_batch_arg(
     ObArray<int64_t> index_array;
     for (int64_t i = 0; OB_SUCC(ret) && i < tablet_arg.table_schemas_.count(); ++i) {
       const share::schema::ObTableSchema *table_schema = tablet_arg.table_schemas_.at(i);
+      const uint64_t tenant_data_version = tablet_arg.tenant_data_version_;
+      const bool need_create_empty_major = tablet_arg.need_create_empty_majors_.at(i);
       int64_t index = OB_INVALID_INDEX;
       if (OB_ISNULL(table_schema)) {
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("table schema is null", KR(ret), K(i), K(tablet_arg));
-      } else if (OB_FAIL(try_add_table_schema(table_schema, index, tablet_arg.compat_mode_))) {
-        LOG_WARN("failed to add table schema to batch", KR(ret), K(table_schema), K(index), K(batch_arg_));
+      } else if (OB_FAIL(try_add_table_schema(table_schema, tenant_data_version,
+          need_create_empty_major, index, tablet_arg.compat_mode_))) {
+        LOG_WARN("failed to add table schema to batch", KR(ret), K(table_schema), K(need_create_empty_major), K(index), K(batch_arg_));
       } else if (OB_UNLIKELY(OB_INVALID_INDEX == index)) {
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("index can not be invalid", KR(ret), K(index), K(tablet_arg), K(batch_arg_));
@@ -168,15 +187,15 @@ int ObBatchCreateTabletHelper::add_arg_to_batch_arg(
   return ret;
 }
 
-int ObBatchCreateTabletHelper::add_table_schema_(const share::schema::ObTableSchema &table_schema,
+int ObBatchCreateTabletHelper::add_table_schema_(
+    const share::schema::ObTableSchema &table_schema,
     const lib::Worker::CompatMode compat_mode,
+    const uint64_t tenant_data_version,
+    const bool need_create_empty_major,
     int64_t &index)
 {
   int ret = OB_SUCCESS;
-  uint64_t data_version = 0;
-  if (OB_FAIL(GET_MIN_DATA_VERSION(table_schema.get_tenant_id(), data_version))) {
-    LOG_WARN("failed to get data version", KR(ret), K(table_schema));
-  } else if (data_version < DATA_VERSION_4_2_2_0) {
+  if (tenant_data_version < DATA_VERSION_4_2_2_0) {
     // compatibility with DATA_VERSION_4_2_1.
     index = batch_arg_.table_schemas_.count();
     if (OB_FAIL(batch_arg_.table_schemas_.push_back(table_schema))) {
@@ -191,7 +210,8 @@ int ObBatchCreateTabletHelper::add_table_schema_(const share::schema::ObTableSch
       LOG_WARN("failed to allocate storage schema", KR(ret), K(table_schema));
     } else if (FALSE_IT(create_tablet_schema = new (create_tablet_schema_ptr)ObCreateTabletSchema())) {
     } else if (OB_FAIL(create_tablet_schema->init(batch_arg_.allocator_, table_schema, compat_mode,
-         false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) {
+         false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
+         tenant_data_version, need_create_empty_major))) {
       LOG_WARN("failed to init storage schema", KR(ret), K(table_schema));
     } else if (OB_FAIL(batch_arg_.create_tablet_schemas_.push_back(create_tablet_schema))) {
       LOG_WARN("failed to push back table schema", KR(ret), K(table_schema));
@@ -202,6 +222,8 @@ int ObBatchCreateTabletHelper::add_table_schema_(const share::schema::ObTableSch
 
 int ObBatchCreateTabletHelper::try_add_table_schema(
     const share::schema::ObTableSchema *table_schema,
+    const uint64_t tenant_data_version,
+    const bool need_create_empty_major,
     int64_t &index,
     const lib::Worker::CompatMode compat_mode)
 {
@@ -219,7 +241,8 @@ int ObBatchCreateTabletHelper::try_add_table_schema(
       if (OB_FAIL(temp_table_schema.assign(*table_schema))) {
         LOG_WARN("failed to assign temp_table_schema", KR(ret), KPC(table_schema));
       } else if (FALSE_IT(temp_table_schema.reset_partition_schema())) {
-      } else if (OB_FAIL(add_table_schema_(temp_table_schema, compat_mode, index))) {
+      } else if (OB_FAIL(add_table_schema_(temp_table_schema, compat_mode,
+          tenant_data_version, need_create_empty_major, index))) {
         LOG_WARN("failed to push back table schema", KR(ret), K(temp_table_schema));
       } else if (OB_FAIL(table_schemas_map_.set_refactored(temp_table_schema.get_table_id(), index))) {
         LOG_WARN("failed to set table schema map", KR(ret), K(index), K(temp_table_schema));

src/rootserver/ob_tablet_creator.h

@@ -37,7 +37,9 @@ public:
      ls_key_(),
      table_schemas_(),
      compat_mode_(lib::Worker::CompatMode::INVALID),
-     is_create_bind_hidden_tablets_(false) {}
+     is_create_bind_hidden_tablets_(false),
+     tenant_data_version_(0),
+     need_create_empty_majors_() {}
   virtual ~ObTabletCreatorArg() {}
   bool is_valid() const;
   void reset();
@@ -47,7 +49,9 @@ public:
            const common::ObTabletID data_tablet_id,
            const ObIArray<const share::schema::ObTableSchema*> &table_schemas,
            const lib::Worker::CompatMode &mode,
-           const bool is_create_bind_hidden_tablets);
+           const bool is_create_bind_hidden_tablets,
+           const uint64_t tenant_data_version,
+           const ObIArray<bool> &need_create_empty_majors);
 
   DECLARE_TO_STRING;
   common::ObArray<common::ObTabletID> tablet_ids_;
@@ -56,6 +60,8 @@ public:
   common::ObArray<const share::schema::ObTableSchema*> table_schemas_;
   lib::Worker::CompatMode compat_mode_;
   bool is_create_bind_hidden_tablets_;
+  uint64_t tenant_data_version_;
+  common::ObArray<bool> need_create_empty_majors_;
 private:
   DISALLOW_COPY_AND_ASSIGN(ObTabletCreatorArg);
 };
@@ -74,6 +80,8 @@ public:
            const share::SCN &major_frozen_scn,
            const bool need_check_tablet_cnt);
   int try_add_table_schema(const share::schema::ObTableSchema *table_schema,
+      const uint64_t tenant_data_version,
+      const bool need_create_empty_major_sstable,
       int64_t &index,
       const lib::Worker::CompatMode compat_mode);
   int add_arg_to_batch_arg(const ObTabletCreatorArg &arg);
@@ -93,6 +101,8 @@ public:
 private:
   int add_table_schema_(const share::schema::ObTableSchema &table_schema,
       const lib::Worker::CompatMode compat_mode,
+      const uint64_t tenant_data_version,
+      const bool need_create_empty_major,
       int64_t &index);
   DISALLOW_COPY_AND_ASSIGN(ObBatchCreateTabletHelper);
 };

src/rootserver/parallel_ddl/ob_create_table_helper.cpp

@@ -2318,6 +2318,8 @@ int ObCreateTableHelper::create_tablets_()
   SCN frozen_scn;
   ObSchemaGetterGuard schema_guard;
   ObSchemaService *schema_service_impl = NULL;
+  uint64_t tenant_data_version = 0;
+
   if (OB_FAIL(check_inner_stat_())) {
     LOG_WARN("fail to check inner stat", KR(ret));
   } else if (OB_ISNULL(schema_service_impl = schema_service_->get_schema_service())) {
@@ -2330,6 +2332,8 @@ int ObCreateTableHelper::create_tablets_()
   } else if (OB_UNLIKELY(new_tables_.count() <= 0)) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("unexpected table cnt", KR(ret), K(new_tables_.count()));
+  } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id_, tenant_data_version))) {
+    LOG_WARN("get min data version failed", K(ret), K_(tenant_id));
   } else {
     ObTableCreator table_creator(
                    tenant_id_,
@@ -2360,6 +2364,7 @@ int ObCreateTableHelper::create_tablets_()
     } else {
       ObArray<const ObTableSchema*> schemas;
       common::ObArray<share::ObLSID> ls_id_array;
+      ObArray<bool> need_create_empty_majors;
       for (int64_t i = 0; OB_SUCC(ret) && i < new_tables_.count(); i++) {
         const ObTableSchema &new_table = new_tables_.at(i);
         const uint64_t table_id = new_table.get_table_id();
@@ -2368,6 +2373,8 @@ int ObCreateTableHelper::create_tablets_()
         } else if (!new_table.is_global_index_table()) {
           if (OB_FAIL(schemas.push_back(&new_table))) {
             LOG_WARN("fail to push back new table", KR(ret));
+          } else if (OB_FAIL(need_create_empty_majors.push_back(true))) {
+            LOG_WARN("fail to push back need create empty major", KR(ret));
           }
         } else {
           if (OB_FAIL(new_table_tablet_allocator.prepare(trans_, new_table))) {
@@ -2375,7 +2382,7 @@ int ObCreateTableHelper::create_tablets_()
           } else if (OB_FAIL(new_table_tablet_allocator.get_ls_id_array(ls_id_array))) {
             LOG_WARN("fail to get ls id array", KR(ret));
           } else if (OB_FAIL(table_creator.add_create_tablets_of_table_arg(
-                     new_table, ls_id_array))) {
+                     new_table, ls_id_array, tenant_data_version, true/*need create major sstable*/))) {
             LOG_WARN("create table partitions failed", KR(ret), K(new_table));
           }
         }
@@ -2395,7 +2402,7 @@ int ObCreateTableHelper::create_tablets_()
         } else if (OB_FAIL(new_table_tablet_allocator.get_ls_id_array(ls_id_array))) {
           LOG_WARN("fail to get ls id array", KR(ret));
         } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg(
-                   schemas, ls_id_array))) {
+                   schemas, ls_id_array, tenant_data_version, need_create_empty_majors /*need create major sstable*/))) {
           LOG_WARN("create table partitions failed", KR(ret), K(data_table));
         } else if (OB_FAIL(table_creator.execute())) {
           LOG_WARN("execute create partition failed", KR(ret));

src/share/datum/ob_datum.h

@@ -162,6 +162,11 @@ struct ObDatumDesc {
   bool is_ext() const { return flag_ == FlagType::EXT; }
   void set_outrow() { null_ = 0; flag_ = FlagType::OUTROW; }
   bool is_outrow() const { return flag_ == FlagType::OUTROW; }
+
+  void set_flag(const FlagType &flag_type) { flag_ = flag_type; }
+  void set_has_lob_header() { flag_ = FlagType::HAS_LOB_HEADER; }
+  bool has_lob_header() const { return flag_ == FlagType::HAS_LOB_HEADER; }
+  void set_flag_none() { flag_ = FlagType::NONE; }
 } __attribute__ ((packed)) ;
 
 // Datum structure, multiple inheritance from ObDatumPtr and ObDatumDesc makes

src/share/diagnosis/ob_sql_monitor_statname.h

@@ -56,6 +56,7 @@ SQL_MONITOR_STATNAME_DEF(SORT_DUMP_DATA_TIME, sql_monitor_statname::INT, "sort d
 // SSTABLE INSERT
 SQL_MONITOR_STATNAME_DEF(DDL_TASK_ID, sql_monitor_statname::INT, "ddl task id", "sort ddl task id")
 SQL_MONITOR_STATNAME_DEF(SSTABLE_INSERT_ROW_COUNT, sql_monitor_statname::INT, "sstable insert row count", "sstable insert row count")
+SQL_MONITOR_STATNAME_DEF(SSTABLE_INSERT_CG_ROW_COUNT, sql_monitor_statname::INT, "sstable insert cg_row count", "sstable insert cg row count")
 // Table Scan stat
 SQL_MONITOR_STATNAME_DEF(IO_READ_BYTES, sql_monitor_statname::CAPACITY, "total io bytes read from disk", "total io bytes read from storage")
 SQL_MONITOR_STATNAME_DEF(TOTAL_READ_BYTES, sql_monitor_statname::CAPACITY, "total bytes processed by storage", "total bytes processed by storage, including memtable")

src/share/inner_table/ob_inner_table_schema.12401_12450.cpp

@@ -8692,6 +8692,538 @@ int ObInnerTableSchema::all_virtual_import_table_task_history_schema(ObTableSche
   return ret;
 }
 
+int ObInnerTableSchema::all_virtual_column_group_mapping_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(4);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("table_id", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_group_id", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_id", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_virtual_column_group_history_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(4);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("table_id", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_group_id", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("schema_version", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("is_deleted", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj column_group_name_default;
+    column_group_name_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("column_group_name", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_COLUMN_GROUP_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false, //is_autoincrement
+      column_group_name_default,
+      column_group_name_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_group_type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("block_size", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("compressor_type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("row_store_type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_virtual_column_group_mapping_history_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(5);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("table_id", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_group_id", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_id", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("schema_version", //column_name
+      ++column_id, //column_id
+      5, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("is_deleted", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 int ObInnerTableSchema::all_virtual_clone_job_schema(ObTableSchema &table_schema)
 {
   int ret = OB_SUCCESS;

src/share/inner_table/ob_inner_table_schema.15401_15450.cpp

@@ -4659,6 +4659,528 @@ int ObInnerTableSchema::all_virtual_ls_info_ora_schema(ObTableSchema &table_sche
   return ret;
 }
 
+int ObInnerTableSchema::all_virtual_column_group_mapping_real_agent_ora_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(4);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN);
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("TENANT_ID", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("TABLE_ID", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_GROUP_ID", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampLTZType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampLTZType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_virtual_column_group_history_ora_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(4);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN);
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("TENANT_ID", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("TABLE_ID", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_GROUP_ID", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("SCHEMA_VERSION", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampLTZType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampLTZType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("IS_DELETED", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_GROUP_NAME", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_UTF8MB4_BIN, //column_collation_type
+      OB_MAX_COLUMN_GROUP_NAME_LENGTH, //column_length
+      2, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_GROUP_TYPE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("BLOCK_SIZE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COMPRESSOR_TYPE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("ROW_STORE_TYPE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_virtual_column_group_mapping_history_ora_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(5);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN);
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("TENANT_ID", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("TABLE_ID", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_GROUP_ID", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("SCHEMA_VERSION", //column_name
+      ++column_id, //column_id
+      5, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampLTZType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampLTZType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("IS_DELETED", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 int ObInnerTableSchema::all_virtual_aux_stat_real_agent_ora_schema(ObTableSchema &table_schema)
 {
   int ret = OB_SUCCESS;

src/share/inner_table/ob_inner_table_schema.21101_21150.cpp

@@ -60,7 +60,7 @@ int ObInnerTableSchema::gv_ob_sstables_schema(ObTableSchema &table_schema)
   table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
 
   if (OB_SUCC(ret)) {
-    if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT  M.SVR_IP,  M.SVR_PORT,  (case M.TABLE_TYPE     when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'     when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'     when 12 then 'MINI' when 13 then 'META'     when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM'     when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG'     else 'INVALID'   end) as TABLE_TYPE,  M.TENANT_ID,  M.LS_ID,  M.TABLET_ID,  M.CG_IDX,  M.START_LOG_SCN,  M.END_LOG_SCN,  M.DATA_CHECKSUM,  M.SIZE,  M.REF,  M.UPPER_TRANS_VERSION,  M.IS_ACTIVE,  M.CONTAIN_UNCOMMITTED_ROW FROM  oceanbase.__all_virtual_table_mgr M )__"))) {
+    if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT  M.SVR_IP,  M.SVR_PORT,  (case M.TABLE_TYPE     when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'     when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'     when 12 then 'MINI' when 13 then 'META'     when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM'     when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE'     else 'INVALID'   end) as TABLE_TYPE,  M.TENANT_ID,  M.LS_ID,  M.TABLET_ID,  M.CG_IDX,  M.START_LOG_SCN,  M.END_LOG_SCN,  M.DATA_CHECKSUM,  M.SIZE,  M.REF,  M.UPPER_TRANS_VERSION,  M.IS_ACTIVE,  M.CONTAIN_UNCOMMITTED_ROW FROM  oceanbase.__all_virtual_table_mgr M )__"))) {
       LOG_ERROR("fail to set view_definition", K(ret));
     }
   }

src/share/inner_table/ob_inner_table_schema.28051_28100.cpp

@@ -310,7 +310,7 @@ int ObInnerTableSchema::gv_ob_sstables_ora_schema(ObTableSchema &table_schema)
   table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
 
   if (OB_SUCC(ret)) {
-    if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT  M.SVR_IP,  M.SVR_PORT,   (case M.TABLE_TYPE     when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'     when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'     when 12 then 'MINI' when 13 then 'META'     when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT'     else 'INVALID'   end) as TABLE_TYPE,  M.LS_ID,  M.TABLET_ID,  M.START_LOG_SCN,  M.END_LOG_SCN,  M."SIZE",  M.REF,  M.UPPER_TRANS_VERSION,  M.IS_ACTIVE,  M.CONTAIN_UNCOMMITTED_ROW FROM  SYS.ALL_VIRTUAL_TABLE_MGR M )__"))) {
+    if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT  M.SVR_IP,  M.SVR_PORT,   (case M.TABLE_TYPE     when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'     when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'     when 12 then 'MINI' when 13 then 'META'     when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE'     else 'INVALID'   end) as TABLE_TYPE,  M.LS_ID,  M.TABLET_ID,  M.START_LOG_SCN,  M.END_LOG_SCN,  M."SIZE",  M.REF,  M.UPPER_TRANS_VERSION,  M.IS_ACTIVE,  M.CONTAIN_UNCOMMITTED_ROW FROM  SYS.ALL_VIRTUAL_TABLE_MGR M )__"))) {
       LOG_ERROR("fail to set view_definition", K(ret));
     }
   }

src/share/inner_table/ob_inner_table_schema.h

@@ -988,6 +988,9 @@ public:
   static int all_virtual_import_table_job_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_import_table_task_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_import_table_task_history_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_column_group_mapping_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_column_group_history_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_column_group_mapping_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_clone_job_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_clone_job_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_aux_stat_schema(share::schema::ObTableSchema &table_schema);
@@ -1251,6 +1254,9 @@ public:
   static int all_virtual_import_table_task_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_import_table_task_history_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_ls_info_ora_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_column_group_mapping_real_agent_ora_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_column_group_history_ora_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_column_group_mapping_history_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_aux_stat_real_agent_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_ls_snapshot_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_index_usage_info_real_agent_ora_schema(share::schema::ObTableSchema &table_schema);
@@ -3551,6 +3557,9 @@ const schema_create_func virtual_table_schema_creators [] = {
   ObInnerTableSchema::all_virtual_import_table_job_history_schema,
   ObInnerTableSchema::all_virtual_import_table_task_schema,
   ObInnerTableSchema::all_virtual_import_table_task_history_schema,
+  ObInnerTableSchema::all_virtual_column_group_mapping_schema,
+  ObInnerTableSchema::all_virtual_column_group_history_schema,
+  ObInnerTableSchema::all_virtual_column_group_mapping_history_schema,
   ObInnerTableSchema::all_virtual_clone_job_schema,
   ObInnerTableSchema::all_virtual_clone_job_history_schema,
   ObInnerTableSchema::all_virtual_aux_stat_schema,
@@ -3824,6 +3833,9 @@ const schema_create_func virtual_table_schema_creators [] = {
   ObInnerTableSchema::all_virtual_import_table_task_ora_schema,
   ObInnerTableSchema::all_virtual_import_table_task_history_ora_schema,
   ObInnerTableSchema::all_virtual_ls_info_ora_schema,
+  ObInnerTableSchema::all_virtual_column_group_mapping_real_agent_ora_schema,
+  ObInnerTableSchema::all_virtual_column_group_history_ora_schema,
+  ObInnerTableSchema::all_virtual_column_group_mapping_history_ora_schema,
   ObInnerTableSchema::all_virtual_aux_stat_real_agent_ora_schema,
   ObInnerTableSchema::all_virtual_ls_snapshot_ora_schema,
   ObInnerTableSchema::all_virtual_index_usage_info_real_agent_ora_schema,
@@ -5324,6 +5336,9 @@ const uint64_t tenant_space_tables [] = {
   OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_HISTORY_TID,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_TID,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID,
   OB_ALL_VIRTUAL_CLONE_JOB_TID,
   OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TID,
   OB_ALL_VIRTUAL_TENANT_SNAPSHOT_JOB_TID,
@@ -5594,6 +5609,9 @@ const uint64_t tenant_space_tables [] = {
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_ORA_TID,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_ORA_TID,
   OB_ALL_VIRTUAL_LS_INFO_ORA_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TID,
   OB_ALL_VIRTUAL_AUX_STAT_REAL_AGENT_ORA_TID,
   OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID,
   OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TID,
@@ -7133,6 +7151,8 @@ const uint64_t all_ora_mapping_virtual_table_org_tables [] = {
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_TID,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_TID,
   OB_ALL_VIRTUAL_LS_INFO_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID,
   OB_ALL_VIRTUAL_LS_SNAPSHOT_TID,  };
 
 const uint64_t all_ora_mapping_virtual_tables [] = {  OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID
@@ -7272,6 +7292,8 @@ const uint64_t all_ora_mapping_virtual_tables [] = {  OB_ALL_VIRTUAL_SQL_AUDIT_O
 ,  OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_ORA_TID
 ,  OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_ORA_TID
 ,  OB_ALL_VIRTUAL_LS_INFO_ORA_TID
+,  OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TID
+,  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TID
 ,  OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID
 ,  };
 
@@ -7757,6 +7779,9 @@ const char* const tenant_space_table_names [] = {
   OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_HISTORY_TNAME,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_TNAME,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_TNAME,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TNAME,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TNAME,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TNAME,
   OB_ALL_VIRTUAL_CLONE_JOB_TNAME,
   OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TNAME,
   OB_ALL_VIRTUAL_TENANT_SNAPSHOT_JOB_TNAME,
@@ -8027,6 +8052,9 @@ const char* const tenant_space_table_names [] = {
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_ORA_TNAME,
   OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_ORA_TNAME,
   OB_ALL_VIRTUAL_LS_INFO_ORA_TNAME,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TNAME,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TNAME,
+  OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TNAME,
   OB_ALL_VIRTUAL_AUX_STAT_REAL_AGENT_ORA_TNAME,
   OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TNAME,
   OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TNAME,
@@ -12201,11 +12229,11 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid,
 
 const int64_t OB_CORE_TABLE_COUNT = 4;
 const int64_t OB_SYS_TABLE_COUNT = 279;
-const int64_t OB_VIRTUAL_TABLE_COUNT = 771;
+const int64_t OB_VIRTUAL_TABLE_COUNT = 777;
 const int64_t OB_SYS_VIEW_COUNT = 826;
-const int64_t OB_SYS_TENANT_TABLE_COUNT = 1881;
+const int64_t OB_SYS_TENANT_TABLE_COUNT = 1887;
 const int64_t OB_CORE_SCHEMA_VERSION = 1;
-const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1884;
+const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1890;
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.vt.cpp

@@ -77,6 +77,13 @@ bool vt_mapping_init()
    tmp_vt_mapping.is_real_vt_ = true;
    }
 
+   {
+   int64_t idx = OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TID - start_idx;
+   VTMapping &tmp_vt_mapping = vt_mappings[idx];
+   tmp_vt_mapping.mapping_tid_ = OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID;
+   tmp_vt_mapping.is_real_vt_ = true;
+   }
+
    {
    int64_t idx = OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_TID - start_idx;
    VTMapping &tmp_vt_mapping = vt_mappings[idx];

src/share/inner_table/ob_inner_table_schema_constants.h

@@ -724,6 +724,9 @@ const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_TID = 12424; // "__all_virtual_im
 const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_HISTORY_TID = 12425; // "__all_virtual_import_table_job_history"
 const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_TID = 12426; // "__all_virtual_import_table_task"
 const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_TID = 12427; // "__all_virtual_import_table_task_history"
+const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID = 12430; // "__all_virtual_column_group_mapping"
+const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID = 12431; // "__all_virtual_column_group_history"
+const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID = 12432; // "__all_virtual_column_group_mapping_history"
 const uint64_t OB_ALL_VIRTUAL_CLONE_JOB_TID = 12435; // "__all_virtual_clone_job"
 const uint64_t OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TID = 12436; // "__all_virtual_clone_job_history"
 const uint64_t OB_ALL_VIRTUAL_AUX_STAT_TID = 12447; // "__all_virtual_aux_stat"
@@ -987,6 +990,9 @@ const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_HISTORY_ORA_TID = 15410; // "ALL_
 const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_ORA_TID = 15411; // "ALL_VIRTUAL_IMPORT_TABLE_TASK_ORA"
 const uint64_t OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_ORA_TID = 15412; // "ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_ORA"
 const uint64_t OB_ALL_VIRTUAL_LS_INFO_ORA_TID = 15414; // "ALL_VIRTUAL_LS_INFO_ORA"
+const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TID = 15417; // "ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA"
+const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TID = 15419; // "ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA"
+const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TID = 15420; // "ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA"
 const uint64_t OB_ALL_VIRTUAL_AUX_STAT_REAL_AGENT_ORA_TID = 15427; // "ALL_VIRTUAL_AUX_STAT_REAL_AGENT_ORA"
 const uint64_t OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID = 15439; // "ALL_VIRTUAL_LS_SNAPSHOT_ORA"
 const uint64_t OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TID = 15440; // "ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA"
@@ -3271,6 +3277,9 @@ const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_TNAME = "__all_virtual_import_
 const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_HISTORY_TNAME = "__all_virtual_import_table_job_history";
 const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_TNAME = "__all_virtual_import_table_task";
 const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_TNAME = "__all_virtual_import_table_task_history";
+const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TNAME = "__all_virtual_column_group_mapping";
+const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TNAME = "__all_virtual_column_group_history";
+const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TNAME = "__all_virtual_column_group_mapping_history";
 const char *const OB_ALL_VIRTUAL_CLONE_JOB_TNAME = "__all_virtual_clone_job";
 const char *const OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TNAME = "__all_virtual_clone_job_history";
 const char *const OB_ALL_VIRTUAL_AUX_STAT_TNAME = "__all_virtual_aux_stat";
@@ -3534,6 +3543,9 @@ const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_JOB_HISTORY_ORA_TNAME = "ALL_VIRTU
 const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_ORA_TNAME = "ALL_VIRTUAL_IMPORT_TABLE_TASK";
 const char *const OB_ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY_ORA_TNAME = "ALL_VIRTUAL_IMPORT_TABLE_TASK_HISTORY";
 const char *const OB_ALL_VIRTUAL_LS_INFO_ORA_TNAME = "ALL_VIRTUAL_LS_INFO";
+const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_COLUMN_GROUP_MAPPING_REAL_AGENT";
+const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_ORA_TNAME = "ALL_VIRTUAL_COLUMN_GROUP_HISTORY";
+const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_ORA_TNAME = "ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY";
 const char *const OB_ALL_VIRTUAL_AUX_STAT_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_AUX_STAT_REAL_AGENT";
 const char *const OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TNAME = "ALL_VIRTUAL_LS_SNAPSHOT";
 const char *const OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT";

src/share/inner_table/ob_inner_table_schema_def.py

@@ -13656,13 +13656,27 @@ def_table_schema(**gen_iterate_private_virtual_table_def(
   table_name = '__all_virtual_import_table_task_history',
   keywords = all_def_keywords['__all_import_table_task_history'],
   in_tenant_space = True))
+
 # 12428: __all_virtual_import_stmt_exec_history
 
 # 12429: __all_virtual_data_activity_metrics
 
-# 12430: __all_virtual_column_group_mapping
-# 12431: __all_virtual_column_group_history
-# 12432: __all_virtual_column_gorup_mapping_history
+def_table_schema(**gen_iterate_virtual_table_def(
+    table_id = '12430',
+    table_name = '__all_virtual_column_group_mapping',
+    keywords = all_def_keywords['__all_column_group_mapping'],
+    in_tenant_space = True))
+def_table_schema(**gen_iterate_virtual_table_def(
+    table_id = '12431',
+    table_name = '__all_virtual_column_group_history',
+    keywords = all_def_keywords['__all_column_group_history'],
+    in_tenant_space = True))
+def_table_schema(**gen_iterate_virtual_table_def(
+    table_id = '12432',
+    table_name = '__all_virtual_column_group_mapping_history',
+    keywords = all_def_keywords['__all_column_group_mapping_history'],
+    in_tenant_space = True))
+
 # 12433: __all_virtual_storage_ha_error_diagnose
 # 12434: __all_virtual_storage_ha_perf_diagnose
 
@@ -14154,11 +14168,11 @@ def_table_schema(**gen_oracle_mapping_virtual_table_def('15414', all_def_keyword
 # 15415: idx_dbms_lock_allocated_lockhandle_real_agent
 # 15416: idx_dbms_lock_allocated_expiration_real_agent
 
-# 15417: __all_virtual_column_group_mapping
+def_table_schema(**gen_oracle_mapping_real_virtual_table_def('15417', all_def_keywords['__all_virtual_column_group_mapping']))
 
 # 15418: __all_virtual_cgroup_config
-# 15419: __all_virutal_column_group_history
-# 15420: __all_virutal_column_group_maping_history
+def_table_schema(**gen_oracle_mapping_virtual_table_def('15419', all_def_keywords['__all_virtual_column_group_history']))
+def_table_schema(**gen_oracle_mapping_virtual_table_def('15420', all_def_keywords['__all_virtual_column_group_mapping_history']))
 # 15421: __all_virtual_wr_system_event
 # 15422: __all_virtual_wr_event_name
 # 15423: __all_tenant_scheduler_running_job
@@ -14186,6 +14200,7 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('1
 # 15442: __all_virtual_column_group
 # 余留位置
 
+
 ################################################################################
 # System View (20000,30000]
 # MySQL System View (20000, 25000]
@@ -16813,7 +16828,7 @@ SELECT
     when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'
     when 12 then 'MINI' when 13 then 'META'
     when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM'
-    when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG'
+    when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE'
     else 'INVALID'
   end) as TABLE_TYPE,
  M.TENANT_ID,
@@ -51818,7 +51833,7 @@ SELECT
     when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'
     when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'
     when 12 then 'MINI' when 13 then 'META'
-    when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT'
+    when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE'
     else 'INVALID'
   end) as TABLE_TYPE,
  M.LS_ID,

src/share/inner_table/ob_inner_table_schema_misc.ipp

@@ -1765,6 +1765,9 @@ case OB_ALL_VIRTUAL_COLL_TYPE_TID:
 case OB_ALL_VIRTUAL_COLL_TYPE_HISTORY_TID:
 case OB_ALL_VIRTUAL_COLUMN_TID:
 case OB_ALL_VIRTUAL_COLUMN_GROUP_TID:
+case OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID:
+case OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID:
+case OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID:
 case OB_ALL_VIRTUAL_COLUMN_HISTORY_TID:
 case OB_ALL_VIRTUAL_COLUMN_STAT_TID:
 case OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_TID:
@@ -2073,6 +2076,51 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       break;
     }
 
+    case OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID: {
+      ObIterateVirtualTable *iter = NULL;
+      if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
+        SERVER_LOG(WARN, "create virtual table iterator failed", K(ret));
+      } else if (OB_FAIL(iter->init(OB_ALL_COLUMN_GROUP_HISTORY_TID, index_schema, params))) {
+        SERVER_LOG(WARN, "virtual table iter init failed", K(ret));
+        iter->~ObIterateVirtualTable();
+        allocator.free(iter);
+        iter = NULL;
+      } else {
+       vt_iter = iter;
+      }
+      break;
+    }
+
+    case OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID: {
+      ObIterateVirtualTable *iter = NULL;
+      if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
+        SERVER_LOG(WARN, "create virtual table iterator failed", K(ret));
+      } else if (OB_FAIL(iter->init(OB_ALL_COLUMN_GROUP_MAPPING_TID, index_schema, params))) {
+        SERVER_LOG(WARN, "virtual table iter init failed", K(ret));
+        iter->~ObIterateVirtualTable();
+        allocator.free(iter);
+        iter = NULL;
+      } else {
+       vt_iter = iter;
+      }
+      break;
+    }
+
+    case OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID: {
+      ObIterateVirtualTable *iter = NULL;
+      if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
+        SERVER_LOG(WARN, "create virtual table iterator failed", K(ret));
+      } else if (OB_FAIL(iter->init(OB_ALL_COLUMN_GROUP_MAPPING_HISTORY_TID, index_schema, params))) {
+        SERVER_LOG(WARN, "virtual table iter init failed", K(ret));
+        iter->~ObIterateVirtualTable();
+        allocator.free(iter);
+        iter = NULL;
+      } else {
+       vt_iter = iter;
+      }
+      break;
+    }
+
     case OB_ALL_VIRTUAL_COLUMN_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2177,7 +2225,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_CONSTRAINT_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2222,9 +2272,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_DAM_LAST_ARCH_TS_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2479,7 +2527,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_FOREIGN_KEY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2524,9 +2574,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_FOREIGN_KEY_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2781,7 +2829,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_MVIEW_REFRESH_CHANGE_STATS_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2826,9 +2876,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_MVIEW_REFRESH_STATS_PARAMS_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3083,7 +3131,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_PENDING_TRANSACTION_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3128,9 +3178,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_RECYCLEBIN_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3385,7 +3433,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_SECURITY_AUDIT_RECORD_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3430,9 +3480,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_SEQUENCE_VALUE_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3687,7 +3735,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TABLEGROUP_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3732,9 +3782,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TABLET_TO_TABLE_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3989,7 +4037,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TENANT_PROFILE_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -4034,9 +4084,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TENANT_SCHEDULER_JOB_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -4291,7 +4339,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TYPE_ATTR_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -4336,9 +4386,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_USER_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {

src/share/ob_ddl_common.cpp

@@ -30,6 +30,7 @@
 #include "storage/tx_storage/ob_ls_map.h"
 #include "rootserver/ob_root_service.h"
 #include "rootserver/ddl_task/ob_ddl_task.h"
+#include "storage/column_store/ob_column_oriented_sstable.h"
 
 using namespace oceanbase::share;
 using namespace oceanbase::common;
@@ -711,7 +712,8 @@ int ObDDLUtil::generate_build_replica_sql(
     const bool use_heap_table_ddl_plan,
     const bool use_schema_version_hint_for_src_table,
     const ObColumnNameMap *col_name_map,
-    ObSqlString &sql_string)
+    ObSqlString &sql_string,
+    const SortCompactLevel compact_level)
 {
   int ret = OB_SUCCESS;
   ObSchemaGetterGuard schema_guard;
@@ -918,8 +920,6 @@ int ObDDLUtil::generate_build_replica_sql(
         }
       }
 
-
-
       if (OB_SUCC(ret)) {
         ObArenaAllocator allocator("ObDDLTmp");
         ObString new_dest_database_name;
@@ -962,8 +962,8 @@ int ObDDLUtil::generate_build_replica_sql(
         }
         if (OB_FAIL(ret)) {
         } else if (oracle_mode) {
-          if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) opt_param('enable_newsort', 'false') use_px */INTO \"%.*s\".\"%.*s\"(%.*s) SELECT /*+ index(\"%.*s\" primary) %.*s */ %.*s from \"%.*s\".\"%.*s\" as of scn %ld %.*s",
-              real_parallelism, execution_id, task_id,
+          if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) opt_param('compact_sort_level', %ld) opt_param('enable_newsort', 'false') use_px */INTO \"%.*s\".\"%.*s\"(%.*s) SELECT /*+ index(\"%.*s\" primary) %.*s */ %.*s from \"%.*s\".\"%.*s\" as of scn %ld %.*s",
+              real_parallelism, execution_id, task_id, static_cast<int64_t>(compact_level),
               static_cast<int>(new_dest_database_name.length()), new_dest_database_name.ptr(), static_cast<int>(new_dest_table_name.length()), new_dest_table_name.ptr(),
               static_cast<int>(insert_column_sql_string.length()), insert_column_sql_string.ptr(),
               static_cast<int>(new_source_table_name.length()), new_source_table_name.ptr(),
@@ -974,8 +974,8 @@ int ObDDLUtil::generate_build_replica_sql(
             LOG_WARN("fail to assign sql string", K(ret));
           }
         } else {
-          if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) opt_param('enable_newsort', 'false') use_px */INTO `%.*s`.`%.*s`(%.*s) SELECT /*+ index(`%.*s` primary) %.*s */ %.*s from `%.*s`.`%.*s` as of snapshot %ld %.*s",
-              real_parallelism, execution_id, task_id,
+          if (OB_FAIL(sql_string.assign_fmt("INSERT /*+ monitor enable_parallel_dml parallel(%ld) opt_param('ddl_execution_id', %ld) opt_param('ddl_task_id', %ld) opt_param('compact_sort_level', %ld), opt_param('enable_newsort', 'false') use_px */INTO `%.*s`.`%.*s`(%.*s) SELECT /*+ index(`%.*s` primary) %.*s */ %.*s from `%.*s`.`%.*s` as of snapshot %ld %.*s",
+              real_parallelism, execution_id, task_id, static_cast<int64_t>(compact_level),
               static_cast<int>(new_dest_database_name.length()), new_dest_database_name.ptr(), static_cast<int>(new_dest_table_name.length()), new_dest_table_name.ptr(),
               static_cast<int>(insert_column_sql_string.length()), insert_column_sql_string.ptr(),
               static_cast<int>(new_source_table_name.length()), new_source_table_name.ptr(),
@@ -1424,12 +1424,17 @@ int64_t ObDDLUtil::get_default_ddl_tx_timeout()
 }
 
 
-int ObDDLUtil::get_data_format_version(
+int ObDDLUtil::get_data_information(
     const uint64_t tenant_id,
     const uint64_t task_id,
-    int64_t &data_format_version)
+    uint64_t &data_format_version,
+    int64_t &snapshot_version,
+    share::ObDDLTaskStatus &task_status)
 {
   int ret = OB_SUCCESS;
+  data_format_version = 0;
+  snapshot_version = 0;
+  task_status = share::ObDDLTaskStatus::PREPARE;
   if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || task_id <= 0
       || nullptr == GCTX.sql_proxy_)) {
     ret = OB_INVALID_ARGUMENT;
@@ -1440,7 +1445,7 @@ int ObDDLUtil::get_data_format_version(
     SMART_VAR(ObMySQLProxy::MySQLResult, res) {
       ObSqlString query_string;
       sqlclient::ObMySQLResult *result = NULL;
-      if (OB_FAIL(query_string.assign_fmt(" SELECT ddl_type, UNHEX(message) as message_unhex FROM %s WHERE task_id = %lu",
+      if (OB_FAIL(query_string.assign_fmt(" SELECT snapshot_version, ddl_type, UNHEX(message) as message_unhex, status FROM %s WHERE task_id = %lu",
           OB_ALL_DDL_TASK_STATUS_TNAME, task_id))) {
         LOG_WARN("assign sql string failed", K(ret));
       } else if (OB_FAIL(GCTX.sql_proxy_->read(res, tenant_id, query_string.ptr()))) {
@@ -1452,10 +1457,14 @@ int ObDDLUtil::get_data_format_version(
         LOG_WARN("get next row failed", K(ret));
       } else {
         int64_t pos = 0;
+        int cur_task_status = 0;
         ObDDLType ddl_type = ObDDLType::DDL_INVALID;
         ObString task_message;
+        EXTRACT_UINT_FIELD_MYSQL(*result, "snapshot_version", snapshot_version, uint64_t);
         EXTRACT_INT_FIELD_MYSQL(*result, "ddl_type", ddl_type, ObDDLType);
         EXTRACT_VARCHAR_FIELD_MYSQL(*result, "message_unhex", task_message);
+        EXTRACT_INT_FIELD_MYSQL(*result, "status", cur_task_status, int);
+        task_status = static_cast<share::ObDDLTaskStatus>(cur_task_status);
         if (ObDDLType::DDL_CREATE_INDEX == ddl_type) {
           SMART_VAR(rootserver::ObIndexBuildTask, task) {
             if (OB_FAIL(task.deserlize_params_from_message(tenant_id, task_message.ptr(), task_message.length(), pos))) {
@@ -2343,6 +2352,145 @@ int ObCheckTabletDataComplementOp::check_and_wait_old_complement_task(
   return ret;
 }
 
+int ObCODDLUtil::get_base_cg_idx(const storage::ObStorageSchema *storage_schema, int64_t &base_cg_idx)
+{
+  int ret = OB_SUCCESS;
+  base_cg_idx = -1;
+  if (OB_UNLIKELY(nullptr == storage_schema)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), KP(storage_schema));
+  } else {
+    bool found_base_cg_idx = false;
+    const ObIArray<ObStorageColumnGroupSchema> &cg_schemas = storage_schema->get_column_groups();
+    for (int64_t i = 0; OB_SUCC(ret) && !found_base_cg_idx && i < cg_schemas.count(); ++i) {
+      const ObStorageColumnGroupSchema &cur_cg_schmea = cg_schemas.at(i);
+      if (cur_cg_schmea.is_all_column_group() || cur_cg_schmea.is_rowkey_column_group()) {
+        base_cg_idx = i;
+        found_base_cg_idx = true;
+      }
+    }
+    if (OB_SUCC(ret) && !found_base_cg_idx) {
+      ret = OB_ENTRY_NOT_EXIST;
+      LOG_WARN("base columng group schema not found", K(ret));
+    }
+  }
+  LOG_DEBUG("get base cg idx", K(ret), K(base_cg_idx));
+  return ret;
+}
+
+int ObCODDLUtil::get_column_checksums(
+    const storage::ObCOSSTableV2 *co_sstable,
+    const storage::ObStorageSchema *storage_schema,
+    ObIArray<int64_t> &column_checksums)
+{
+  int ret = OB_SUCCESS;
+  column_checksums.reset();
+  int64_t column_count = 0;
+  if (OB_UNLIKELY(nullptr == co_sstable || nullptr == storage_schema)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), KP(co_sstable), KP(storage_schema));
+  } else if (OB_FAIL(storage_schema->get_stored_column_count_in_sstable(column_count))) {
+    LOG_WARN("fail to get_stored_column_count_in_sstable", K(ret), KPC(storage_schema));
+  } else {
+    const common::ObIArray<ObStorageColumnGroupSchema> &column_groups = storage_schema->get_column_groups();
+    ObArray<bool/*checksum_ready*/> checksum_ready_array;
+    if (OB_FAIL(checksum_ready_array.reserve(column_count))) {
+      LOG_WARN("reserve checksum ready array failed", K(ret), K(column_count));
+    } else if (OB_FAIL(column_checksums.reserve(column_count))) {
+      LOG_WARN("reserve checksum array failed", K(ret), K(column_count));
+    }
+    for (int64_t i = 0; i < column_count && OB_SUCC(ret); i ++) {
+      if (OB_FAIL(checksum_ready_array.push_back(false))) {
+        LOG_WARN("push back ready flag failed", K(ret), K(i));
+      } else if (OB_FAIL(column_checksums.push_back(0))) {
+        LOG_WARN("fail to push back column checksum", K(ret), K(i));
+      }
+    }
+    ObSSTableWrapper cg_sstable_wrapper;
+    ObSSTable *cg_sstable = nullptr;
+    for (int64_t i = 0; !co_sstable->is_empty_co_table() && i < column_groups.count() && OB_SUCC(ret); i++) {
+      const ObStorageColumnGroupSchema &column_group = column_groups.at(i);
+      ObSSTableMetaHandle cg_table_meta_hdl;
+      if (column_group.is_all_column_group()) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("unexpected column_group", K(ret), K(i));
+      } else if (OB_FAIL(co_sstable->fetch_cg_sstable(i, cg_sstable_wrapper))) {
+        LOG_WARN("fail to get cg sstable", K(ret), K(i));
+      } else if (OB_FAIL(cg_sstable_wrapper.get_sstable(cg_sstable))) {
+        LOG_WARN("get sstable failed", K(ret));
+      } else if (OB_UNLIKELY(cg_sstable == nullptr || !cg_sstable->is_valid())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("unexpec cg sstable", K(ret), KPC(cg_sstable));
+      } else if (OB_FAIL(cg_sstable->get_meta(cg_table_meta_hdl))) {
+        LOG_WARN("fail to get meta", K(ret), KPC(cg_sstable));
+      } else if (OB_UNLIKELY(cg_table_meta_hdl.get_sstable_meta().get_col_checksum_cnt() != column_group.get_column_count())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("unexpected col_checksum_cnt", K(ret),
+            K(cg_table_meta_hdl.get_sstable_meta().get_col_checksum_cnt()), K(column_group.get_column_count()));
+      } else {
+        for (int64_t j = 0; j < column_group.get_column_count() && OB_SUCC(ret); j++) {
+          const uint16_t column_idx = column_group.column_idxs_[j];
+          if (column_idx < 0 || column_idx >= column_checksums.count()) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("invalid column index", K(ret), K(i), K(j), K(column_idx), K(column_checksums.count()));
+          } else {
+            int64_t &column_checksum = column_checksums.at(column_idx);
+            bool &is_checksum_ready = checksum_ready_array.at(column_idx);
+            if (!is_checksum_ready) {
+              column_checksum = cg_table_meta_hdl.get_sstable_meta().get_col_checksum()[j];
+              is_checksum_ready = true;
+            } else if (OB_UNLIKELY(column_checksum != cg_table_meta_hdl.get_sstable_meta().get_col_checksum()[j])) {
+              ret = OB_ERR_UNEXPECTED;
+              LOG_WARN("unexpected col_checksum_cnt", K(ret), K(column_checksum), K(cg_table_meta_hdl.get_sstable_meta().get_col_checksum()[j]));
+            }
+          }
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObCODDLUtil::is_rowkey_based_co_sstable(
+    const storage::ObCOSSTableV2 *co_sstable,
+    const storage::ObStorageSchema *storage_schema,
+    bool &is_rowkey_based)
+{
+  int ret = OB_SUCCESS;
+  is_rowkey_based = false;
+  if (OB_UNLIKELY(nullptr == co_sstable || nullptr == storage_schema)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), KP(co_sstable), KP(storage_schema));
+  } else {
+    const int64_t base_cg_idx = co_sstable->get_key().get_column_group_id();
+    if (base_cg_idx < 0 || base_cg_idx >= storage_schema->get_column_groups().count()) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("invalid base column group index", K(ret), K(base_cg_idx));
+    } else {
+      is_rowkey_based = storage_schema->get_column_groups().at(base_cg_idx).is_rowkey_column_group();
+    }
+  }
+  return ret;
+}
+
+
+
+int ObCODDLUtil::need_column_group_store(const storage::ObStorageSchema &table_schema, bool &need_column_group)
+{
+  int ret = OB_SUCCESS;
+  need_column_group = table_schema.get_column_group_count() > 1;
+  return ret;
+}
+
+int ObCODDLUtil::need_column_group_store(const schema::ObTableSchema &table_schema, bool &need_column_group)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(table_schema.get_is_column_store(need_column_group))) {
+      SHARE_LOG(WARN, "fail to check whether table is column store", K(ret));
+  }
+  return ret;
+}
+
 //record trace_id
 ObDDLEventInfo::ObDDLEventInfo()
   : addr_(GCTX.self_addr()),

src/share/ob_ddl_common.h

@@ -31,6 +31,7 @@ struct ObDropTableArg;
 struct ObDropIndexArg;
 struct ObTruncateTableArg;
 struct ObCreateIndexArg;
+struct ObIndexArg;
 }
 namespace sql
 {
@@ -41,6 +42,8 @@ namespace storage
 {
 class ObTabletHandle;
 class ObLSHandle;
+struct ObStorageColumnGroupSchema;
+class ObCOSSTableV2;
 }
 namespace share
 {
@@ -89,6 +92,7 @@ enum ObDDLType
   DDL_TABLE_RESTORE = 1013, // table restore
   DDL_MVIEW_COMPLETE_REFRESH = 1014,
   DDL_CREATE_MVIEW = 1015,
+  DDL_ALTER_COLUMN_GROUP = 1016, // alter table add/drop column group
 
   // @note new normal ddl type to be defined here !!!
   DDL_NORMAL_TYPE = 10001,
@@ -143,6 +147,16 @@ enum ObDDLTaskStatus {
   SUCCESS = 100
 };
 
+enum SortCompactLevel
+{
+  SORT_DEFAULT_LEVEL = 0,
+  SORT_COMPACT_LEVEL = 1,
+  SORT_ENCODE_LEVEL = 2,
+  SORT_COMPRESSION_LEVEL = 3,
+  SORT_COMPRESSION_COMPACT_LEVEL = 4,
+  SORT_COMPRESSION_ENCODE_LEVEL = 5
+};
+
 static const char* ddl_task_status_to_str(const ObDDLTaskStatus &task_status) {
   const char *str = nullptr;
   switch(task_status) {
@@ -375,7 +389,8 @@ public:
       const bool use_heap_table_ddl_plan,
       const bool use_schema_version_hint_for_src_table,
       const ObColumnNameMap *col_name_map,
-      ObSqlString &sql_string);
+      ObSqlString &sql_string,
+      const share::SortCompactLevel compact_level = share::SORT_DEFAULT_LEVEL);
 
   static int generate_build_mview_replica_sql(
       const uint64_t tenant_id,
@@ -422,7 +437,7 @@ public:
       storage::ObLSHandle &ls_handle,
       const ObTabletID &tablet_id,
       storage::ObTabletHandle &tablet_handle,
-      const storage::ObMDSGetTabletMode mode = storage::ObMDSGetTabletMode::READ_READABLE_COMMITED);
+      const storage::ObMDSGetTabletMode mode = storage::ObMDSGetTabletMode::READ_WITHOUT_CHECK);
 
   static int clear_ddl_checksum(sql::ObPhysicalPlan *phy_plan);
   
@@ -467,10 +482,12 @@ public:
   static int64_t get_default_ddl_rpc_timeout();
   static int64_t get_default_ddl_tx_timeout();
 
-  static int get_data_format_version(
+  static int get_data_information(
      const uint64_t tenant_id,
      const uint64_t task_id,
-     int64_t &data_format_version);
+     uint64_t &data_format_version,
+     int64_t &snapshot_version,
+     share::ObDDLTaskStatus &task_status);
 
   static int replace_user_tenant_id(
     const ObDDLType &ddl_type,
@@ -549,6 +566,28 @@ private:
       uint64_t &table_id);
 };
 
+class ObCODDLUtil
+{
+public:
+  static int need_column_group_store(const storage::ObStorageSchema &table_schema, bool &need_column_group);
+  static int need_column_group_store(const schema::ObTableSchema &table_schema, bool &need_column_group);
+
+  static int get_base_cg_idx(
+      const storage::ObStorageSchema *storage_schema,
+      int64_t &base_cg_idx);
+
+  static int get_column_checksums(
+      const storage::ObCOSSTableV2 *co_sstable,
+      const storage::ObStorageSchema *storage_schema,
+      ObIArray<int64_t> &column_checksums);
+
+  static int is_rowkey_based_co_sstable(
+      const storage::ObCOSSTableV2 *co_sstable,
+      const storage::ObStorageSchema *storage_schema,
+      bool &is_rowkey_based);
+};
+
+
 
 class ObCheckTabletDataComplementOp
 {

src/share/ob_ddl_error_message_table_operator.cpp

@@ -16,6 +16,7 @@
 #include "share/inner_table/ob_inner_table_schema.h"
 #include "share/ob_get_compat_mode.h"
 #include "share/schema/ob_schema_utils.h"
+#include "share/schema/ob_table_param.h"
 #include "share/ob_ddl_sim_point.h"
 
 using namespace oceanbase::share;
@@ -103,7 +104,7 @@ int ObDDLErrorMessageTableOperator::get_index_task_info(
 }
 
 int ObDDLErrorMessageTableOperator::extract_index_key(const ObTableSchema &index_schema,
-    const ObStoreRowkey &index_key, char *buffer, const int64_t buffer_len)
+    const blocksstable::ObDatumRowkey &index_key, char *buffer, const int64_t buffer_len)
 {
   int ret = OB_SUCCESS;
   if (!index_schema.is_valid() || !index_key.is_valid() || OB_ISNULL(buffer) || buffer_len <= 0) {
@@ -112,33 +113,33 @@ int ObDDLErrorMessageTableOperator::extract_index_key(const ObTableSchema &index
   } else {
     const int64_t index_size = index_schema.get_index_column_num();
     int64_t pos = 0;
-    int64_t valid_index_size = 0;
-    uint64_t column_id = OB_INVALID_ID;
     MEMSET(buffer, 0, buffer_len);
-
     for (int64_t i = 0; OB_SUCC(ret) && i < index_size; i++) {
-      if (OB_FAIL(index_schema.get_index_info().get_column_id(i, column_id))) {
+      const ObRowkeyColumn *column = index_schema.get_index_info().get_column(i);
+      if (OB_ISNULL(column)) {
+        ret = OB_ERR_UNEXPECTED;
         LOG_WARN("Failed to get index column description", K(i), K(ret));
-      } else if (column_id <= OB_MIN_SHADOW_COLUMN_ID) {
-        valid_index_size ++;
-      }
-    }
-    for (int64_t i = 0; OB_SUCC(ret) && i < valid_index_size; ++i) {
-      const ObObj &obj  = index_key.get_obj_ptr()[i];
-      if (OB_FAIL(obj.print_plain_str_literal(buffer, buffer_len, pos))) {
-        LOG_WARN("fail to print_plain_str_literal", K(ret), KP(buffer));
-      } else if (i < valid_index_size - 1) {
-        if (OB_FAIL(databuff_printf(buffer,  buffer_len, pos, "-"))) {
+      } else if (IS_SHADOW_COLUMN(column->column_id_)) {
+        break;
+      } else {
+        const blocksstable::ObStorageDatum &datum = index_key.get_datum(i);
+        ObObj obj;
+        if (OB_FAIL(datum.to_obj(obj, column->get_meta_type()))) {
+          LOG_WARN("convert datum to obj failed", K(ret));
+        } else if (OB_FAIL(obj.print_plain_str_literal(buffer, buffer_len, pos))) {
+          LOG_WARN("fail to print_plain_str_literal", K(ret), KP(buffer));
+        } else if (OB_FAIL(databuff_printf(buffer,  buffer_len, pos, "-"))) {
           LOG_WARN("databuff print failed", K(ret));
         }
       }
     }
-    if (buffer != nullptr) {
-      buffer[pos++] = '\0';
-      if (OB_SIZE_OVERFLOW == ret) {
-        LOG_WARN("the index key length is larger than OB_TMP_BUF_SIZE_256", K(index_key), KP(buffer));
-        ret = OB_SUCCESS;
-      }
+    if (OB_SUCC(ret) && pos > 0) {
+      buffer[pos - 1] = '\0'; // overwrite the tail '-'
+    }
+    if (OB_SIZE_OVERFLOW == ret) {
+      buffer[buffer_len - 1] = '\0';
+      LOG_WARN("the index key length is larger than OB_TMP_BUF_SIZE_256", K(index_key), KP(buffer));
+      ret = OB_SUCCESS;
     }
   }
 

src/share/ob_ddl_error_message_table_operator.h

@@ -21,6 +21,7 @@
 #include "share/schema/ob_schema_struct.h"
 #include "share/ob_ddl_common.h"
 #include "share/schema/ob_table_schema.h"
+#include "storage/blocksstable/ob_datum_rowkey.h"
 
 namespace oceanbase
 {
@@ -87,7 +88,7 @@ public:
   ObDDLErrorMessageTableOperator();
   virtual ~ObDDLErrorMessageTableOperator();
   static int get_index_task_info(ObMySQLProxy &sql_proxy, const share::schema::ObTableSchema &index_schema, ObDDLErrorInfo &info);
-  static int extract_index_key(const share::schema::ObTableSchema &index_schema, const common::ObStoreRowkey &index_key,
+  static int extract_index_key(const share::schema::ObTableSchema &index_schema, const blocksstable::ObDatumRowkey &index_key,
     char *buffer, const int64_t buffer_len);
   static int load_ddl_user_error(const uint64_t tenant_id, const int64_t task_id, const uint64_t table_id, 
       common::ObMySQLProxy &sql_proxy, ObBuildDDLErrorMessage &error_message);

src/share/ob_ddl_task_executor.h

@@ -77,7 +77,7 @@ private:
         || common::OB_TRANS_NEED_ROLLBACK == ret_code || common::OB_RPC_SEND_ERROR == ret_code || common::OB_RPC_CONNECT_ERROR == ret_code
         || common::OB_RPC_POST_ERROR == ret_code || common::OB_TRANS_ROLLBACKED == ret_code || common::OB_TRANS_KILLED == ret_code
         || common::OB_GET_LOCATION_TIME_OUT == ret_code || common::OB_TRANS_RPC_TIMEOUT == ret_code || common::OB_LIBEASY_ERROR == ret_code
-        || common::OB_TRANS_CTX_NOT_EXIST == ret_code;
+        || common::OB_TRANS_CTX_NOT_EXIST == ret_code || OB_ERR_SESSION_INTERRUPTED == ret_code;
   }
   static bool is_retry(const int ret_code) {
     return common::OB_EAGAIN == ret_code || common::OB_DDL_SCHEMA_VERSION_NOT_MATCH == ret_code || common::OB_TASK_EXPIRED == ret_code || common::OB_NEED_RETRY == ret_code

src/share/ob_errno.cpp

@@ -1924,10 +1924,10 @@ static const _error _error_OB_COLUMN_GROUP_NOT_FOUND = {
       .mysql_errno           = -1,
       .sqlstate              = "HY000",
       .str_error             = "Column group not found",
-      .str_user_error        = "Column group not found",
+      .str_user_error        = "Column group \'%.*s\' not found",
       .oracle_errno          = 600,
       .oracle_str_error      = "ORA-00600: internal error code, arguments: -4185, Column group not found",
-      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -4185, Column group not found"
+      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -4185, Column grou \'%.*s\' not found"
 };
 static const _error _error_OB_CS_COMPRESS_LIB_ERROR = {
       .error_name            = "OB_CS_COMPRESS_LIB_ERROR",

src/share/ob_errno.def

@@ -253,7 +253,7 @@ DEFINE_ERROR_DEP(OB_ERR_ALREADY_EXISTS, -4181, -1, "42S01", "Already exist");
 DEFINE_ERROR_DEP(OB_SEARCH_NOT_FOUND, -4182, -1, "HY000", "Value not found");
 DEFINE_ERROR(OB_BEYOND_THE_RANGE, -4183, -1, "HY000", "Key out of range");
 DEFINE_ERROR(OB_SERVER_OUTOF_DISK_SPACE, -4184, -1, "53100", "Server out of disk space");
-DEFINE_ERROR(OB_COLUMN_GROUP_NOT_FOUND, -4185, -1, "HY000", "Column group not found");
+DEFINE_ERROR(OB_COLUMN_GROUP_NOT_FOUND, -4185, -1, "HY000", "Column group not found", "Column group \'%.*s\' not found");
 DEFINE_ERROR(OB_CS_COMPRESS_LIB_ERROR, -4186, -1, "HY000", "Server failed to get compress library");
 DEFINE_ERROR_DEP(OB_ITEM_NOT_MATCH, -4187, -1, "HY000", "Item not match");
 DEFINE_ERROR(OB_SCHEDULER_TASK_CNT_MISMATCH, -4188, -1, "HY000", "Running task cnt and unfinished task cnt not consistent");

src/share/ob_errno.h

@@ -2005,7 +2005,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_SEARCH_NOT_FOUND__USER_ERROR_MSG "Value not found"
 #define OB_BEYOND_THE_RANGE__USER_ERROR_MSG "Key out of range"
 #define OB_SERVER_OUTOF_DISK_SPACE__USER_ERROR_MSG "Server out of disk space"
-#define OB_COLUMN_GROUP_NOT_FOUND__USER_ERROR_MSG "Column group not found"
+#define OB_COLUMN_GROUP_NOT_FOUND__USER_ERROR_MSG "Column group \'%.*s\' not found"
 #define OB_CS_COMPRESS_LIB_ERROR__USER_ERROR_MSG "Server failed to get compress library"
 #define OB_ITEM_NOT_MATCH__USER_ERROR_MSG "Item not match"
 #define OB_SCHEDULER_TASK_CNT_MISMATCH__USER_ERROR_MSG "Running task cnt and unfinished task cnt not consistent"
@@ -4215,7 +4215,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_SEARCH_NOT_FOUND__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4182, Value not found"
 #define OB_BEYOND_THE_RANGE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4183, Key out of range"
 #define OB_SERVER_OUTOF_DISK_SPACE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4184, Server out of disk space"
-#define OB_COLUMN_GROUP_NOT_FOUND__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4185, Column group not found"
+#define OB_COLUMN_GROUP_NOT_FOUND__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4185, Column group %.*s not found"
 #define OB_CS_COMPRESS_LIB_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4186, Server failed to get compress library"
 #define OB_ITEM_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4187, Item not match"
 #define OB_SCHEDULER_TASK_CNT_MISMATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4188, Running task cnt and unfinished task cnt not consistent"

src/share/ob_rpc_struct.cpp

@@ -2960,7 +2960,8 @@ DEF_TO_STRING(ObIndexArg)
        K_(index_name),
        K_(table_name),
        K_(database_name),
-       K_(index_action_type));
+       K_(index_action_type),
+       K_(compact_level));
   J_OBJ_END();
   return pos;
 }
@@ -2971,7 +2972,8 @@ OB_SERIALIZE_MEMBER((ObIndexArg, ObDDLArg),
                     table_name_,
                     database_name_,
                     index_action_type_,
-                    session_id_);
+                    session_id_,
+                    compact_level_);
 
 bool ObCreateIndexArg::is_valid() const
 {
@@ -2983,6 +2985,17 @@ bool ObCreateIndexArg::is_valid() const
          && index_using_type_ >= USING_BTREE
          && index_using_type_ < USING_TYPE_MAX;
 }
+OB_SERIALIZE_MEMBER(ObCreateIndexArg::ObIndexColumnGroupItem, is_each_cg_, column_list_);
+
+int ObCreateIndexArg::ObIndexColumnGroupItem::assign(const ObCreateIndexArg::ObIndexColumnGroupItem &other)
+{
+  int ret = OB_SUCCESS;
+  is_each_cg_ = other.is_each_cg_;
+  if (OB_FAIL(column_list_.assign(other.column_list_))) {
+    LOG_WARN("fail to assign array", K(ret));
+  }
+  return ret;
+}
 
 DEF_TO_STRING(ObCreateIndexArg)
 {
@@ -3008,7 +3021,9 @@ DEF_TO_STRING(ObCreateIndexArg)
        K_(nls_timestamp_tz_format),
        K_(sql_mode),
        K_(inner_sql_exec_addr),
-       K_(local_session_var));
+       K_(local_session_var),
+       K_(exist_all_column_group),
+       K_(index_cgs));
   J_OBJ_END();
   return pos;
 }
@@ -3032,7 +3047,9 @@ OB_SERIALIZE_MEMBER((ObCreateIndexArg, ObIndexArg),
                     nls_timestamp_tz_format_,
                     sql_mode_,
                     inner_sql_exec_addr_,
-                    local_session_var_);
+                    local_session_var_,
+                    exist_all_column_group_,
+                    index_cgs_);
 
 bool ObAlterIndexArg::is_valid() const
 {
@@ -5298,6 +5315,24 @@ bool ObUpdateIndexStatusArg::is_valid() const
       && status_ < INDEX_STATUS_MAX;
 }
 
+int ObUpdateIndexStatusArg::assign(const ObUpdateIndexStatusArg &other_arg)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(ObDDLArg::assign(other_arg))) {
+    LOG_WARN("assign other arg failed", K(ret));
+  } else {
+    index_table_id_ = other_arg.index_table_id_;
+    status_ = other_arg.status_;
+    convert_status_ = other_arg.convert_status_;
+    in_offline_ddl_white_list_ = other_arg.in_offline_ddl_white_list_;
+    data_table_id_ = other_arg.data_table_id_;
+    database_name_ = other_arg.database_name_;
+    task_id_ = other_arg.task_id_;
+    error_code_ = other_arg.error_code_;
+  }
+  return ret;
+}
+
 int ObUpdateMViewStatusArg::assign(const ObUpdateMViewStatusArg &other)
 {
   int ret = OB_SUCCESS;
@@ -5326,8 +5361,10 @@ OB_SERIALIZE_MEMBER((ObUpdateIndexStatusArg, ObDDLArg),
                     status_,
                     convert_status_,
                     in_offline_ddl_white_list_,
-		                data_table_id_,
-                    database_name_);
+                    data_table_id_,
+                    database_name_,
+                    task_id_,
+                    error_code_);
 
 OB_SERIALIZE_MEMBER((ObUpdateMViewStatusArg, ObDDLArg),
                     mview_table_id_,

src/share/ob_rpc_struct.h

@@ -1254,6 +1254,7 @@ public:
   common::ObString table_name_;
   common::ObString database_name_;
   IndexActionType index_action_type_;
+  share::SortCompactLevel compact_level_;
 
   ObIndexArg():
       ObDDLArg(),
@@ -1262,7 +1263,8 @@ public:
       index_name_(),
       table_name_(),
       database_name_(),
-      index_action_type_(INVALID_ACTION)
+      index_action_type_(INVALID_ACTION),
+      compact_level_(share::SORT_COMPACT_LEVEL)
   {}
   virtual ~ObIndexArg() {}
   void reset()
@@ -1273,6 +1275,7 @@ public:
     table_name_.reset();
     database_name_.reset();
     index_action_type_ = INVALID_ACTION;
+    compact_level_ = share::SORT_COMPACT_LEVEL;
     ObDDLArg::reset();
   }
   bool is_valid() const;
@@ -1288,6 +1291,7 @@ public:
       table_name_ = other.table_name_;
       database_name_ = other.database_name_;
       index_action_type_ = other.index_action_type_;
+      compact_level_ = other.compact_level_;
     }
     return ret;
   }
@@ -2376,7 +2380,7 @@ public:
   DECLARE_TO_STRING;
 };
 
-struct ObColumnSortItem
+struct ObColumnSortItem final
 {
   OB_UNIS_VERSION(1);
 public:
@@ -2502,7 +2506,9 @@ public:
         sql_mode_(0),
         inner_sql_exec_addr_(),
         allocator_(),
-        local_session_var_(&allocator_)
+        local_session_var_(&allocator_),
+        exist_all_column_group_(false),
+        index_cgs_()
   {
     index_action_type_ = ADD_INDEX;
     index_using_type_ = share::schema::USING_BTREE;
@@ -2532,6 +2538,8 @@ public:
     inner_sql_exec_addr_.reset();
     local_session_var_.reset();
     allocator_.reset();
+    exist_all_column_group_ = false;
+    index_cgs_.reset();
   }
   void set_index_action_type(const IndexActionType type) { index_action_type_  = type; }
   bool is_valid() const;
@@ -2547,6 +2555,8 @@ public:
       SHARE_LOG(WARN, "fail to assign hidden store columns", K(ret));
     } else if (OB_FAIL(fulltext_columns_.assign(other.fulltext_columns_))) {
       SHARE_LOG(WARN, "fail to assign fulltext columns", K(ret));
+    } else if (OB_FAIL(index_cgs_.assign(other.index_cgs_))) {
+      SHARE_LOG(WARN, "fail to assign index cgs", K(ret));
     } else if (OB_FAIL(index_schema_.assign(other.index_schema_))) {
       SHARE_LOG(WARN, "fail to assign index schema", K(ret));
     } else if (OB_FAIL(local_session_var_.deep_copy(other.local_session_var_))){
@@ -2566,6 +2576,7 @@ public:
       sql_mode_ = other.sql_mode_;
       inner_sql_exec_addr_ = other.inner_sql_exec_addr_;
       consumer_group_id_ = other.consumer_group_id_;
+      exist_all_column_group_ = other.exist_all_column_group_;
     }
     return ret;
   }
@@ -2581,6 +2592,38 @@ public:
                                                 || share::schema::INDEX_TYPE_SPATIAL_GLOBAL == index_type_
                                                 || share::schema::INDEX_TYPE_SPATIAL_GLOBAL_LOCAL_STORAGE == index_type_; }
 
+//todo @qilu:only for each_cg now, when support customized cg ,refine this
+  typedef common::ObSEArray<uint64_t, common::DEFAULT_CUSTOMIZED_CG_NUM> ObCGColumnList;
+  struct ObIndexColumnGroupItem final
+  {
+    OB_UNIS_VERSION(1);
+  public:
+    ObIndexColumnGroupItem() : is_each_cg_(false), column_list_()
+    {}
+    ObIndexColumnGroupItem(const bool is_each_cg) : is_each_cg_(is_each_cg), column_list_()
+    {}
+    ~ObIndexColumnGroupItem()
+    {
+      reset();
+    }
+    bool is_valid() const
+    {
+      return is_each_cg_;
+    }
+    void reset()
+    {
+      is_each_cg_ = false;
+      column_list_.reset();
+    }
+    int assign(const ObIndexColumnGroupItem &other);
+    TO_STRING_KV(K(is_each_cg_), K(column_list_));
+
+  public:
+    bool is_each_cg_;
+    ObCGColumnList column_list_;
+  };
+
+public:
   share::schema::ObIndexType index_type_;
   common::ObSEArray<ObColumnSortItem, common::OB_PREALLOCATED_NUM> index_columns_;
   common::ObSEArray<common::ObString, common::OB_PREALLOCATED_NUM> store_columns_;
@@ -2602,7 +2645,8 @@ public:
   common::ObAddr inner_sql_exec_addr_;
   common::ObArenaAllocator allocator_;
   ObLocalSessionVar local_session_var_;
-
+  bool exist_all_column_group_;
+  common::ObSEArray<ObIndexColumnGroupItem, 1/*each*/> index_cgs_;
 };
 
 typedef ObCreateIndexArg ObAlterPrimaryArg;
@@ -6051,13 +6095,16 @@ public:
     convert_status_(true),
     in_offline_ddl_white_list_(false),
     data_table_id_(common::OB_INVALID_ID),
-    database_name_()
+    database_name_(),
+    task_id_(0),
+    error_code_(OB_SUCCESS)
   {}
   bool is_valid() const;
   virtual bool is_allow_when_disable_ddl() const;
   virtual bool is_allow_when_upgrade() const { return true; }
   virtual bool is_in_offline_ddl_white_list() const { return in_offline_ddl_white_list_; }
-  TO_STRING_KV(K_(index_table_id), K_(status), K_(convert_status), K_(in_offline_ddl_white_list), K_(data_table_id), K_(database_name));
+  int assign(const ObUpdateIndexStatusArg &other_arg);
+  TO_STRING_KV(K_(index_table_id), K_(status), K_(convert_status), K_(in_offline_ddl_white_list), K_(task_id), K_(error_code), K_(data_table_id), K_(database_name));
 
   uint64_t index_table_id_;
   share::schema::ObIndexStatus status_;
@@ -6065,6 +6112,8 @@ public:
   bool in_offline_ddl_white_list_;
   uint64_t data_table_id_;
   ObString database_name_;
+  int64_t task_id_;
+  int error_code_;
 };
 
 struct ObUpdateMViewStatusArg : public ObDDLArg
@@ -9306,7 +9355,7 @@ public:
   int64_t parallelism_;
   int64_t execution_id_;
   int64_t tablet_task_id_;
-  int64_t data_format_version_;
+  uint64_t data_format_version_;
   int64_t consumer_group_id_;
   uint64_t dest_tenant_id_;
   share::ObLSID dest_ls_id_;

src/share/ob_tablet_autoincrement_param.cpp

@@ -41,6 +41,16 @@ int ObTabletCacheInterval::next_value(uint64_t &next_value)
   return ret;
 }
 
+int ObTabletCacheInterval::get_value(uint64_t &value)
+{
+  int ret = OB_SUCCESS;
+  value = next_value_;
+  if (value + 1 > end_) {
+    ret = OB_EAGAIN;
+  }
+  return ret;
+}
+
 int ObTabletCacheInterval::fetch(uint64_t count, ObTabletCacheInterval &dest)
 {
   int ret = OB_SUCCESS;

src/share/ob_tablet_autoincrement_param.h

@@ -106,8 +106,10 @@ public:
   TO_STRING_KV(K_(tablet_id), K_(start), K_(end), K_(cache_size), K_(next_value), K_(task_id));
   void set(uint64_t start, uint64_t end);
   int next_value(uint64_t &next_value);
+  int get_value(uint64_t &value);
   int fetch(uint64_t count, ObTabletCacheInterval &dest);
   uint64_t count() const { return end_ - start_ + 1; }
+  uint64_t remain_count() const { return end_ - next_value_ + 1; }
   bool operator <(const ObTabletCacheInterval &other) { return tablet_id_ < other.tablet_id_; }
 public:
   common::ObTabletID tablet_id_;