fix wrong row_offset in ddl_cg_sstable && fix invalid cg_iter_param

mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h

@@ -746,7 +746,7 @@ void TestIndexBlockDataPrepare::prepare_ddl_kv()
         macro_handle.set_block_id(data_macro_meta.get_macro_id());
         ObDataMacroBlockMeta *copied_meta = nullptr;
         ASSERT_EQ(OB_SUCCESS, data_macro_meta.deep_copy(copied_meta, allocator_));
-        ASSERT_EQ(OB_SUCCESS, ddl_kv_.insert_block_meta_tree(macro_handle, copied_meta));
+        ASSERT_EQ(OB_SUCCESS, ddl_kv_.insert_block_meta_tree(macro_handle, copied_meta, 0));
       }
     }
     ASSERT_EQ(OB_ITER_END, ret);
@@ -1207,7 +1207,7 @@ void TestIndexBlockDataPrepare::prepare_merge_ddl_kvs()
         if (macro_idx > partial_kv_start_idx_) {
           ObDataMacroBlockMeta *copied_meta = nullptr;
           ASSERT_EQ(OB_SUCCESS, data_macro_meta.deep_copy(copied_meta, allocator_));
-          ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->get_ddl_memtables().at(kv_idx)->insert_block_meta_tree(macro_handle, copied_meta));
+          ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->get_ddl_memtables().at(kv_idx)->insert_block_meta_tree(macro_handle, copied_meta, 0));
           ++kv_idx;
         }
       }

src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp

@@ -27,7 +27,7 @@ namespace blocksstable
 ObDDLIndexBlockRowIterator::ObDDLIndexBlockRowIterator()
   : is_iter_start_(false),
     is_iter_finish_(true),
-    is_normal_cg_(false),
+    is_co_sstable_(false),
     btree_iter_(),
     block_meta_tree_(nullptr),
     cur_tree_value_(nullptr)
@@ -45,7 +45,7 @@ void ObDDLIndexBlockRowIterator::reset()
   ObIndexBlockRowIterator::reset();
   is_iter_finish_ = true;
   is_iter_start_ = false;
-  is_normal_cg_ = false;
+  is_co_sstable_ = false;
   btree_iter_.reset();
   block_meta_tree_ = nullptr;
   cur_tree_value_ = nullptr;
@@ -55,7 +55,7 @@ void ObDDLIndexBlockRowIterator::reuse()
 {
   is_iter_finish_ = true;
   is_iter_start_ = false;
-  is_normal_cg_ = false;
+  is_co_sstable_ = false;
   btree_iter_.reset();
   block_meta_tree_ = nullptr;
   cur_tree_value_ = nullptr;
@@ -76,7 +76,7 @@ int ObDDLIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data,
     is_reverse_scan_ = is_reverse_scan;
     iter_step_ = is_reverse_scan_ ? -1 : 1;
     datum_utils_ = datum_utils;
-    is_normal_cg_ = iter_param.is_valid() ? iter_param.sstable_->is_normal_cg_sstable() : false;
+    is_co_sstable_ = iter_param.is_valid() ? iter_param.sstable_->is_co_sstable() : false;
     is_inited_ = true;
   }
   return ret;
@@ -85,7 +85,7 @@ int ObDDLIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data,
 int ObDDLIndexBlockRowIterator::set_iter_param(const ObStorageDatumUtils *datum_utils,
                                                bool is_reverse_scan,
                                                const storage::ObBlockMetaTree *block_meta_tree,
-                                               const bool is_normal_cg,
+                                               const bool is_co_sstable,
                                                const int64_t iter_step)
 {
   int ret = OB_SUCCESS;
@@ -97,7 +97,7 @@ int ObDDLIndexBlockRowIterator::set_iter_param(const ObStorageDatumUtils *datum_
     is_reverse_scan_ = is_reverse_scan;
     iter_step_ = iter_step == INT64_MAX ? (is_reverse_scan_ ? -1 : 1) : iter_step;
     datum_utils_ = datum_utils;
-    is_normal_cg_ = is_normal_cg;
+    is_co_sstable_ = is_co_sstable;
     is_inited_ = true;
   }
   return ret;
@@ -312,6 +312,30 @@ int ObDDLIndexBlockRowIterator::get_current(const ObIndexBlockRowHeader *&idx_ro
   return ret;
 }
 
+int ObDDLIndexBlockRowIterator::inner_get_current(const ObIndexBlockRowHeader *&idx_row_header,
+                                                  const ObDatumRowkey *&endkey,
+                                                  int64_t &row_offset)
+{
+  int ret = OB_SUCCESS;
+  bool is_start_key = false;
+  bool is_end_key = false;
+  idx_row_header = nullptr;
+  endkey = nullptr;
+  row_offset = 0;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("Iter not opened yet", K(ret), KPC(this));
+  } else if (OB_ISNULL(cur_tree_value_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("cur tree value is null", K(ret));
+  } else {
+    idx_row_header = &(cur_tree_value_->header_);
+    endkey = &(cur_tree_value_->block_meta_->end_key_);
+    row_offset = cur_tree_value_->co_sstable_row_offset_;
+  }
+  return ret;
+}
+
 int ObDDLIndexBlockRowIterator::get_next(const ObIndexBlockRowHeader *&idx_row_header,
                                          const ObDatumRowkey *&endkey,
                                          bool &is_scan_left_border,
@@ -332,11 +356,12 @@ int ObDDLIndexBlockRowIterator::get_next(const ObIndexBlockRowHeader *&idx_row_h
   row_offset = 0;
   bool is_start_key = false;
   bool is_end_key = false;
+  int64_t co_sstable_row_offset = 0;
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
     LOG_WARN("Iter not opened yet", K(ret), KPC(this));
-  } else if (OB_FAIL(get_current(idx_row_header, endkey))) {
+  } else if (OB_FAIL(inner_get_current(idx_row_header, endkey, co_sstable_row_offset))) {
-    LOG_WARN("read cur idx row failed", K(ret), KPC(idx_row_header), KPC(endkey));
+    LOG_WARN("read cur idx row failed", K(ret), KPC(idx_row_header), KPC(endkey), K(co_sstable_row_offset));
   } else if (OB_UNLIKELY(nullptr == idx_row_header || nullptr == endkey)) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("Unexpected null index block row header/endkey", K(ret), KP(idx_row_header), KP(endkey));
@@ -345,6 +370,8 @@ int ObDDLIndexBlockRowIterator::get_next(const ObIndexBlockRowHeader *&idx_row_h
                          !idx_row_header->is_major_node())) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("invalid index row header", K(ret), KPC(idx_row_header));
+  } else {
+    row_offset = is_co_sstable_ ? co_sstable_row_offset : 0;
   }
 
   if (OB_SUCC(ret)) {
@@ -368,7 +395,6 @@ int ObDDLIndexBlockRowIterator::get_next(const ObIndexBlockRowHeader *&idx_row_h
       cur_tree_value_ = tmp_tree_value;
     }
     if (OB_SUCC(ret)) {
-      row_offset = is_normal_cg_ ?  0 : endkey->datums_[0].get_int();
       is_scan_left_border = is_reverse_scan_ ? is_end_key : is_start_key;
       is_scan_right_border = is_reverse_scan_ ? is_start_key : is_end_key;
     }

src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h

@@ -64,19 +64,23 @@ public:
   virtual void reuse() override;
   virtual void set_iter_end() override { is_iter_finish_ = true; }
   INHERIT_TO_STRING_KV("base iterator:", ObIndexBlockRowIterator, "format:", "ObDDLIndexBlockRowIterator",
-                       K_(is_iter_start), K_(is_iter_finish), KP(cur_tree_value_), KP(block_meta_tree_), K(is_normal_cg_));
+                       K_(is_iter_start), K_(is_iter_finish), KP(cur_tree_value_), KP(block_meta_tree_), K(is_co_sstable_));
 public:
   int set_iter_param(const ObStorageDatumUtils *datum_utils,
                      bool is_reverse_scan,
                      const storage::ObBlockMetaTree *block_meta_tree,
-                     const bool is_normal_cg,
+                     const bool is_co_sstable,
                      const int64_t iter_step = INT64_MAX);
   bool is_valid() { return OB_NOT_NULL(block_meta_tree_); }
   int get_next_meta(const ObDataMacroBlockMeta *&meta);
+private:
+  int inner_get_current(const ObIndexBlockRowHeader *&idx_row_header,
+                        const ObDatumRowkey *&endkey,
+                        int64_t &row_offset/*for co sstable*/);
 private:
   bool is_iter_start_;
   bool is_iter_finish_;
-  bool is_normal_cg_;
+  bool is_co_sstable_;
   DDLBtreeIterator btree_iter_;
   const storage::ObBlockMetaTree *block_meta_tree_;
   storage::ObBlockMetaTreeValue *cur_tree_value_;

src/storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.cpp

@@ -103,7 +103,7 @@ int ObSSTableSecMetaIterator::open(
   }
   if (OB_FAIL(ret) || is_prefetch_end_) {
   } else if (is_ddl_mem_sstable) {
-    const bool is_normal_cg = sstable.is_normal_cg_sstable();
+    const bool is_co_sstable = sstable.is_co_sstable();
     const ObMicroBlockData &root_block = sstable_meta_hdl_.get_sstable_meta().get_root_info().get_block_data();
     if (ObMicroBlockData::DDL_BLOCK_TREE != root_block.type_ || nullptr == root_block.buf_) {
       ret = OB_ERR_UNEXPECTED;
@@ -111,7 +111,7 @@ int ObSSTableSecMetaIterator::open(
     } else {
       block_meta_tree_ = reinterpret_cast<ObBlockMetaTree *>(const_cast<char *>(root_block.buf_));
       const int64_t step = max(1, sample_step);
-      if (OB_FAIL(ddl_iter_.set_iter_param(const_cast<ObStorageDatumUtils *>(&rowkey_read_info.get_datum_utils()), is_reverse_scan, block_meta_tree_, is_normal_cg, step))) {
+      if (OB_FAIL(ddl_iter_.set_iter_param(const_cast<ObStorageDatumUtils *>(&rowkey_read_info.get_datum_utils()), is_reverse_scan, block_meta_tree_, is_co_sstable, step))) {
         LOG_WARN("fail to set ddl iter param", K(ret));
       } else if (OB_FAIL(ddl_iter_.locate_range(query_range,
                                                 true, /*is_left_border*/

src/storage/column_store/ob_cg_iter_param_pool.cpp

@@ -75,6 +75,7 @@ int ObCGIterParamPool::get_iter_param(
         LOG_WARN("Unexpected null iter param", K(ret));
       } else if (tmp_param->can_be_reused(cg_idx, exprs, is_aggregate)) {
         iter_param = tmp_param;
+        iter_param->get_table_param_ = row_param.get_table_param_;
         break;
       }
     }
@@ -199,6 +200,7 @@ int ObCGIterParamPool::fill_virtual_cg_iter_param(
     cg_param.output_exprs_ = output_exprs;
     cg_param.op_ = row_param.op_;
     cg_param.pd_storage_flag_ = row_param.pd_storage_flag_;
+    cg_param.get_table_param_ = row_param.get_table_param_;
   }
   if (OB_FAIL(ret) && nullptr != output_exprs) {
     output_exprs->reset();
@@ -253,6 +255,7 @@ int ObCGIterParamPool::generate_for_column_store(const ObTableIterParam &row_par
     cg_param.tablet_id_ = row_param.tablet_id_;
     cg_param.cg_idx_ = cg_idx;
     cg_param.read_info_ = cg_param.cg_read_info_handle_.get_read_info();
+    cg_param.get_table_param_ = row_param.get_table_param_;
     cg_param.cg_col_param_ = col_param;
     cg_param.out_cols_project_ = out_cols_project;
     cg_param.agg_cols_project_ = nullptr;

src/storage/ddl/ob_ddl_merge_task.cpp

@@ -818,7 +818,7 @@ int get_sorted_meta_array(
                   LOG_WARN("hold macro block failed", K(ret));
                 } else if (OB_FAIL(data_macro_meta.deep_copy(copied_meta, allocator))) {
                   LOG_WARN("deep copy macro block meta failed", K(ret));
-                } else if (OB_FAIL(meta_tree.insert_macro_block(macro_handle, &copied_meta->end_key_, copied_meta))) {
+                } else if (OB_FAIL(meta_tree.insert_macro_block(macro_handle, &copied_meta->end_key_, copied_meta))) { // useless co_sstable_row_offset
                   LOG_WARN("insert meta tree failed", K(ret), K(macro_handle), KPC(copied_meta));
                   copied_meta->~ObDataMacroBlockMeta();
                 } else {

src/storage/ddl/ob_tablet_ddl_kv.cpp

@@ -264,7 +264,8 @@ void ObBlockMetaTree::destroy_tree_value()
 
 int ObBlockMetaTree::insert_macro_block(const ObDDLMacroHandle &macro_handle,
                                         const blocksstable::ObDatumRowkey *rowkey,
-                                        const blocksstable::ObDataMacroBlockMeta *meta)
+                                        const blocksstable::ObDataMacroBlockMeta *meta,
+                                        const int64_t co_sstable_row_offset)
 {
   int ret = OB_SUCCESS;
   ObDataMacroBlockMeta *insert_meta = const_cast<ObDataMacroBlockMeta *>(meta);
@@ -284,6 +285,7 @@ int ObBlockMetaTree::insert_macro_block(const ObDDLMacroHandle &macro_handle,
   } else {
     tree_value = new (buf) ObBlockMetaTreeValue(insert_meta, rowkey);
 
+    tree_value->co_sstable_row_offset_ = co_sstable_row_offset;
     tree_value->header_.version_ = ObIndexBlockRowHeader::INDEX_BLOCK_HEADER_V1;
     tree_value->header_.row_store_type_ = static_cast<uint8_t>(data_desc_.get_desc().get_row_store_type());
     tree_value->header_.compressor_type_ = static_cast<uint8_t>(data_desc_.get_desc().get_compressor_type());
@@ -879,11 +881,11 @@ int ObDDLMemtable::init_ddl_index_iterator(const blocksstable::ObStorageDatumUti
                                            blocksstable::ObDDLIndexBlockRowIterator *ddl_kv_index_iter)
 {
   int ret = OB_SUCCESS;
-  const bool is_normal_cg = is_normal_cg_sstable();
+  const bool is_co_sst = is_co_sstable();
   if (OB_ISNULL(datum_utils) || OB_UNLIKELY(!datum_utils->is_valid()) || OB_ISNULL(ddl_kv_index_iter)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid arguement", K(ret), KP(ddl_kv_index_iter), KPC(datum_utils));
-  } else if (OB_FAIL(ddl_kv_index_iter->set_iter_param(datum_utils, is_reverse_scan, &block_meta_tree_, is_normal_cg))) {
+  } else if (OB_FAIL(ddl_kv_index_iter->set_iter_param(datum_utils, is_reverse_scan, &block_meta_tree_, is_co_sst))) {
     LOG_WARN("fail to set ddl iter param", K(ret));
   }
   return ret;
@@ -1142,7 +1144,7 @@ int ObDDLKV::set_macro_block(
         }
       }
       if (OB_FAIL(ret)) {
-      } else if (OB_FAIL(ddl_memtable->insert_block_meta_tree(macro_block.block_handle_, data_macro_meta))) {
+      } else if (OB_FAIL(ddl_memtable->insert_block_meta_tree(macro_block.block_handle_, data_macro_meta, macro_block.end_row_id_))) {
         LOG_WARN("insert block meta tree faield", K(ret));
       } else {
         min_scn_ = SCN::min(min_scn_, macro_block.scn_);
@@ -1155,10 +1157,10 @@ int ObDDLKV::set_macro_block(
   return ret;
 }
 
-int ObDDLMemtable::insert_block_meta_tree(const ObDDLMacroHandle &macro_handle, blocksstable::ObDataMacroBlockMeta *data_macro_meta)
+int ObDDLMemtable::insert_block_meta_tree(const ObDDLMacroHandle &macro_handle, blocksstable::ObDataMacroBlockMeta *data_macro_meta, const int64_t co_sstable_row_offset)
 {
   int ret = OB_SUCCESS;
-  if (OB_FAIL(block_meta_tree_.insert_macro_block(macro_handle, &data_macro_meta->end_key_, data_macro_meta))) {
+  if (OB_FAIL(block_meta_tree_.insert_macro_block(macro_handle, &data_macro_meta->end_key_, data_macro_meta, co_sstable_row_offset))) {
     LOG_WARN("insert macro block failed", K(ret), K(macro_handle), KPC(data_macro_meta));
   } else {
     const ObDataBlockMetaVal &meta_val = data_macro_meta->get_meta_val();

src/storage/ddl/ob_tablet_ddl_kv.h

@@ -44,18 +44,20 @@ namespace storage
 class ObBlockMetaTreeValue final
 {
 public:
-  ObBlockMetaTreeValue() : block_meta_(nullptr), rowkey_(nullptr), header_() {}
+  ObBlockMetaTreeValue() : co_sstable_row_offset_(0), block_meta_(nullptr), rowkey_(nullptr), header_() {}
   ObBlockMetaTreeValue(const blocksstable::ObDataMacroBlockMeta *block_meta,
                        const blocksstable::ObDatumRowkey *rowkey)
-    : block_meta_(block_meta), rowkey_(rowkey), header_(){}
+    : co_sstable_row_offset_(0), block_meta_(block_meta), rowkey_(rowkey), header_(){}
   ~ObBlockMetaTreeValue()
   {
+    co_sstable_row_offset_ = 0;
     block_meta_ = nullptr;
     rowkey_ = nullptr;
   }
-  TO_STRING_KV(KPC_(block_meta), KPC_(rowkey), K_(header));
+  TO_STRING_KV(K_(co_sstable_row_offset), KPC_(block_meta), KPC_(rowkey), K_(header));
 
 public:
+  int64_t co_sstable_row_offset_;
   const blocksstable::ObDataMacroBlockMeta *block_meta_;
   const blocksstable::ObDatumRowkey *rowkey_;
   blocksstable::ObIndexBlockRowHeader header_;
@@ -78,7 +80,8 @@ public:
   void destroy_tree_value();
   int insert_macro_block(const ObDDLMacroHandle &macro_handle,
                          const blocksstable::ObDatumRowkey *rowkey,
-                         const blocksstable::ObDataMacroBlockMeta *meta);
+                         const blocksstable::ObDataMacroBlockMeta *meta,
+                         const int64_t co_sstable_row_offset = 0);
   int locate_key(const blocksstable::ObDatumRange &range,
                  const blocksstable::ObStorageDatumUtils &datum_utils,
                  blocksstable::DDLBtreeIterator &iter,
@@ -161,7 +164,8 @@ public:
   void reset();
   int insert_block_meta_tree(
       const ObDDLMacroHandle &macro_handle,
-      blocksstable::ObDataMacroBlockMeta *data_macro_meta);
+      blocksstable::ObDataMacroBlockMeta *data_macro_meta,
+      const int64_t co_sstable_row_offset);
   void set_scn_range(
       const share::SCN &start_scn,
       const share::SCN &end_scn);