fix backup report -4002 issue

src/storage/backup/ob_backup_utils.cpp

@@ -1126,17 +1126,52 @@ ObBackupProviderItem::ObBackupProviderItem()
 ObBackupProviderItem::~ObBackupProviderItem()
 {}
 
+int ObBackupProviderItem::set_with_fake(const ObBackupProviderItemType &item_type, const common::ObTabletID &tablet_id)
+{
+  int ret = OB_SUCCESS;
+  if (PROVIDER_ITEM_SSTABLE_META != item_type && PROVIDER_ITEM_TABLET_META != item_type) {
+    ret = OB_ERR_SYS;
+    LOG_WARN("get invalid args", K(ret));
+  } else if (!tablet_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("get invalid args", K(ret), K(tablet_id));
+  } else {
+    item_type_ = item_type;
+    logic_id_ = get_fake_logic_id_();
+    macro_block_id_ = get_fake_macro_id_();
+    table_key_ = get_fake_table_key_();
+    tablet_id_ = tablet_id;
+    if (!is_valid()) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("provider item not valid", K(ret), KPC(this));
+    }
+  }
+  return ret;
+}
+
 int ObBackupProviderItem::set(const ObBackupProviderItemType &item_type, const ObBackupMacroBlockId &macro_id,
      const ObITable::TableKey &table_key, const common::ObTabletID &tablet_id)
 {
   int ret = OB_SUCCESS;
-  item_type_ = item_type;
+  if (PROVIDER_ITEM_MACRO_ID != item_type) {
-  logic_id_ = macro_id.logic_id_;
+    ret = OB_ERR_SYS;
-  macro_block_id_ = macro_id.macro_block_id_;
+    LOG_WARN("get invalid args", K(ret));
-  table_key_ = table_key;
+  } else if (!macro_id.is_valid() || !table_key.is_valid() || !tablet_id.is_valid()) {
-  tablet_id_ = tablet_id;
+    ret = OB_INVALID_ARGUMENT;
-  nested_offset_ = macro_id.nested_offset_;
+    LOG_WARN("get invalid args", K(ret), K(macro_id), K(table_key), K(tablet_id));
-  nested_size_ = macro_id.nested_size_;
+  } else {
+    item_type_ = item_type;
+    logic_id_ = macro_id.logic_id_;
+    macro_block_id_ = macro_id.macro_block_id_;
+    table_key_ = table_key;
+    tablet_id_ = tablet_id;
+    nested_offset_ = macro_id.nested_offset_;
+    nested_size_ = macro_id.nested_size_;
+    if (!is_valid()) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("provider item not valid", K(ret), KPC(this));
+    }
+  }
   return ret;
 }
 
@@ -1208,15 +1243,14 @@ int ObBackupProviderItem::deep_copy(const ObBackupProviderItem &src, char *buf,
 
 bool ObBackupProviderItem::is_valid() const
 {
-  bool bret = true;
+  bool bret = false;
-  if (PROVIDER_ITEM_MACRO_ID == item_type_) {
+  if (PROVIDER_ITEM_MACRO_ID != item_type_
-    bret = logic_id_.is_valid() && table_key_.is_valid() && tablet_id_.is_valid(); // no need to check macro id valid
+      && PROVIDER_ITEM_SSTABLE_META != item_type_
-  } else if (PROVIDER_ITEM_SSTABLE_META == item_type_) {
+      && PROVIDER_ITEM_TABLET_META != item_type_) {
-    bret = tablet_id_.is_valid();
-  } else if (PROVIDER_ITEM_TABLET_META == item_type_) {
-    bret = tablet_id_.is_valid();
-  } else {
     bret = false;
+  } else {
+    bret = logic_id_.is_valid() && macro_block_id_.is_valid()
+        && table_key_.is_valid() && tablet_id_.is_valid();
   }
   return bret;
 }
@@ -1290,6 +1324,26 @@ DEFINE_GET_SERIALIZE_SIZE(ObBackupProviderItem)
   return size;
 }
 
+ObITable::TableKey ObBackupProviderItem::get_fake_table_key_() const
+{
+  ObITable::TableKey table_key;
+  table_key.tablet_id_ = ObTabletID(1);
+  table_key.table_type_ = ObITable::TableType::MAJOR_SSTABLE;
+  table_key.version_range_.snapshot_version_ = 0;
+  table_key.column_group_idx_ = 0;
+  return table_key;
+}
+
+ObLogicMacroBlockId ObBackupProviderItem::get_fake_logic_id_() const
+{
+  return ObLogicMacroBlockId(0/*data_seq*/, 1/*logic_version*/, 1/*tablet_id*/);
+}
+
+MacroBlockId ObBackupProviderItem::get_fake_macro_id_() const
+{
+  return MacroBlockId(4096/*first_id*/, 0/*second_id*/, 0/*third_id*/);
+}
+
 /* ObBackupProviderItemCompare */
 
 ObBackupProviderItemCompare::ObBackupProviderItemCompare(int &sort_ret) : result_code_(sort_ret), backup_data_type_()
@@ -1874,13 +1928,7 @@ int ObBackupTabletProvider::add_sstable_item_(const common::ObTabletID &tablet_i
 {
   int ret = OB_SUCCESS;
   ObBackupProviderItem item;
-  ObLogicMacroBlockId fake_logic_id;
+  if (OB_FAIL(item.set_with_fake(PROVIDER_ITEM_SSTABLE_META, tablet_id))) {
-  MacroBlockId fake_macro_block_id;
-  ObITable::TableKey fake_table_key;
-  ObBackupMacroBlockId macro_id;
-  macro_id.macro_block_id_ = fake_macro_block_id;
-  macro_id.logic_id_ = fake_logic_id;
-  if (OB_FAIL(item.set(PROVIDER_ITEM_SSTABLE_META, macro_id, fake_table_key, tablet_id))) {
     LOG_WARN("failed to set item", K(ret), K(tablet_id));
   } else if (!item.is_valid()) {
     ret = OB_INVALID_DATA;
@@ -1897,14 +1945,8 @@ int ObBackupTabletProvider::add_tablet_item_(const common::ObTabletID &tablet_id
 {
   int ret = OB_SUCCESS;
   ObBackupProviderItem item;
-  ObLogicMacroBlockId fake_logic_id;
+  if (OB_FAIL(item.set_with_fake(PROVIDER_ITEM_TABLET_META, tablet_id))) {
-  MacroBlockId fake_macro_block_id;
+    LOG_WARN("failed to set item", K(ret), K(tablet_id));
-  ObITable::TableKey fake_table_key;
-  ObBackupMacroBlockId macro_id;
-  macro_id.macro_block_id_ = fake_macro_block_id;
-  macro_id.logic_id_ = fake_logic_id;
-  if (OB_FAIL(item.set(PROVIDER_ITEM_TABLET_META, macro_id, fake_table_key, tablet_id))) {
-    LOG_WARN("failed to set item", K(ret), K(fake_table_key), K(tablet_id));
   } else if (!item.is_valid()) {
     ret = OB_INVALID_DATA;
     LOG_WARN("backup item is not valid", K(ret), K(item));

src/storage/backup/ob_backup_utils.h

@@ -179,6 +179,9 @@ class ObBackupProviderItem {
 public:
   ObBackupProviderItem();
   virtual ~ObBackupProviderItem();
+  // for tablet meta and sstable meta
+  int set_with_fake(const ObBackupProviderItemType &item_type, const common::ObTabletID &tablet_id);
+  // for macro block
   int set(const ObBackupProviderItemType &item_type, const ObBackupMacroBlockId &backup_macro_id,
       const storage::ObITable::TableKey &table_key, const common::ObTabletID &tablet_id);
   bool operator==(const ObBackupProviderItem &other) const;
@@ -196,6 +199,11 @@ public:
   void reset();
   TO_STRING_KV(K_(item_type), K_(logic_id), K_(table_key), K_(tablet_id), K_(nested_offset), K_(nested_size));
   NEED_SERIALIZE_AND_DESERIALIZE;
+private:
+  // for parallel external sort serialization restriction
+  ObITable::TableKey get_fake_table_key_() const;
+  blocksstable::ObLogicMacroBlockId get_fake_logic_id_() const;
+  blocksstable::MacroBlockId get_fake_macro_id_() const;
 
 private:
   ObBackupProviderItemType item_type_;

unittest/storage/backup/test_backup_utils.cpp

@@ -21,6 +21,8 @@
 #include "test_backup.h"
 #include "storage/backup/ob_backup_utils.h"
 #include "storage/blocksstable/ob_logic_macro_id.h"
+#include "storage/ob_parallel_external_sort.h"
+#include "storage/blocksstable/ob_data_file_prepare.h"
 
 using namespace oceanbase;
 using namespace oceanbase::common;
@@ -31,6 +33,8 @@ using namespace oceanbase::blocksstable;
 namespace oceanbase {
 namespace backup {
 
+static ObSimpleMemLimitGetter getter;
+
 class ObFakeBackupTabletProvider : public ObIBackupTabletProvider {
 public:
   ObFakeBackupTabletProvider();
@@ -280,6 +284,143 @@ TEST(TestBackupUtils, test_check_macro_block_reuse)
   ASSERT_FALSE(need_skip);
 }
 
+class TestBackupExternalSort : public blocksstable::TestDataFilePrepare
+{
+public:
+  TestBackupExternalSort();
+  virtual ~TestBackupExternalSort() {}
+  int init_tenant_mgr();
+  void destroy_tenant_mgr();
+  int64_t calculate_min_item_count();
+  int generate_items(const int64_t min_count, common::ObIAllocator &allocator, ObVector<ObBackupProviderItem *> &items);
+  virtual void SetUp();
+  virtual void TearDown();
+public:
+  static const int64_t MACRO_BLOCK_SIZE = 2 * 1024 * 1024;
+  static const int64_t MACRO_BLOCK_COUNT = 15 * 1024;
+private:
+  common::ObArenaAllocator allocator_;
+};
+
+TestBackupExternalSort::TestBackupExternalSort()
+  : TestDataFilePrepare(&getter, "TestBackupExternalSort", MACRO_BLOCK_SIZE, MACRO_BLOCK_COUNT),
+    allocator_(ObModIds::TEST)
+{
+}
+
+void TestBackupExternalSort::SetUp()
+{
+  TestDataFilePrepare::SetUp();
+  EXPECT_EQ(OB_SUCCESS, init_tenant_mgr());
+  EXPECT_EQ(OB_SUCCESS, ObTmpFileManager::get_instance().init());
+  static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
+  ObTenantEnv::set_tenant(&tenant_ctx);
+  ObTenantIOManager *io_service = nullptr;
+  EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
+}
+
+void TestBackupExternalSort::TearDown()
+{
+  allocator_.reuse();
+  ObTmpFileManager::get_instance().destroy();
+  TestDataFilePrepare::TearDown();
+  destroy_tenant_mgr();
+}
+
+int TestBackupExternalSort::init_tenant_mgr()
+{
+  int ret = OB_SUCCESS;
+  ObAddr self;
+  self.set_ip_addr("127.0.0.1", 8086);
+  rpc::frame::ObReqTransport req_transport(NULL, NULL);
+  const int64_t ulmt = 128LL << 30;
+  const int64_t llmt = 128LL << 30;
+  ret = getter.add_tenant(OB_SYS_TENANT_ID, ulmt, llmt);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = getter.add_tenant(OB_SERVER_TENANT_ID, ulmt, llmt);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  lib::set_memory_limit(128LL << 32);
+  return ret;
+}
+
+void TestBackupExternalSort::destroy_tenant_mgr()
+{
+}
+
+int64_t TestBackupExternalSort::calculate_min_item_count()
+{
+  ObBackupProviderItem item;
+  const int64_t min_memory_limit = ObExternalSortConstant::MIN_MEMORY_LIMIT;
+  const int64_t item_size = sizeof(item) + item.get_deep_copy_size();
+  int64_t min_count = min_memory_limit / item_size;
+  LOG_INFO("calculate min item count", K(min_memory_limit), K(item_size), K(min_count));
+  return min_count;
+}
+
+int TestBackupExternalSort::generate_items(const int64_t min_count, common::ObIAllocator &allocator, ObVector<ObBackupProviderItem *> &items)
+{
+  int ret = OB_SUCCESS;
+  ObBackupProviderItem *item = NULL;
+  items.reset();
+  if (min_count < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(WARN, "invalid argument", K(ret), K(min_count));
+  } else {
+    void *buf = NULL;
+    ObTabletID tablet_id;
+    for (int64_t i = 0; OB_SUCC(ret) && i < min_count; ++i) {
+      ObBackupProviderItemType item_type = PROVIDER_ITEM_TABLET_META;
+      make_random_tablet_id(tablet_id);
+      if (OB_ISNULL(buf = allocator.alloc(sizeof(ObBackupProviderItem)))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_WARN("failed to allocate memory", K(ret));
+      } else if (OB_ISNULL(item = new (buf) ObBackupProviderItem())) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_WARN("failed to placement new item", K(ret));
+      } else if (OB_FAIL(item->set_with_fake(item_type, tablet_id))) {
+        LOG_WARN("failed to set item", K(ret), K(tablet_id));
+      } else if (OB_FAIL(items.push_back(item))) {
+        LOG_WARN("failed to push back item", K(ret));
+      }
+    }
+  }
+  return ret;
+}
+
+TEST_F(TestBackupExternalSort, test_sort)
+{
+  int ret = OB_SUCCESS;
+  int64_t buf_mem_limit = ObExternalSortConstant::MIN_MEMORY_LIMIT;
+  int64_t file_buf_size = 2 << 20;
+  int64_t expire_timestamp = 0;
+  const uint64_t tenant_id = OB_SYS_TENANT_ID;
+  typedef storage::ObExternalSort<ObBackupProviderItem, ObBackupProviderItemCompare> ExternalSort;
+  ExternalSort external_sort;
+  ObBackupProviderItemCompare backup_item_cmp(ret);
+  ObBackupDataType backup_data_type;
+  backup_data_type.set_major_data_backup();
+  backup_item_cmp.set_backup_data_type(backup_data_type);
+
+  ObArenaAllocator allocator;
+  const int64_t min_count = calculate_min_item_count();
+  ObVector<ObBackupProviderItem *> total_items;
+
+  ret = generate_items(min_count, allocator, total_items);
+  EXPECT_EQ(OB_SUCCESS, ret);
+
+  ret = external_sort.init(buf_mem_limit, file_buf_size, expire_timestamp, tenant_id, &backup_item_cmp);
+  EXPECT_EQ(OB_SUCCESS, ret);
+
+  for (int64_t i = 0; OB_SUCC(ret) && i < total_items.size(); ++i) {
+    const ObBackupProviderItem *item = total_items.at(i);
+    ret = external_sort.add_item(*item);
+    EXPECT_EQ(OB_SUCCESS, ret);
+  }
+
+  ret = external_sort.do_sort(true /*final_merge*/);
+  EXPECT_EQ(OB_SUCCESS, ret);
+}
+
 }  // namespace backup
 }  // namespace oceanbase