fix backup report -4002 issue

unittest/storage/backup/test_backup_utils.cpp

@@ -21,6 +21,8 @@
 #include "test_backup.h"
 #include "storage/backup/ob_backup_utils.h"
 #include "storage/blocksstable/ob_logic_macro_id.h"
+#include "storage/ob_parallel_external_sort.h"
+#include "storage/blocksstable/ob_data_file_prepare.h"
 
 using namespace oceanbase;
 using namespace oceanbase::common;
@@ -31,6 +33,8 @@ using namespace oceanbase::blocksstable;
 namespace oceanbase {
 namespace backup {
 
+static ObSimpleMemLimitGetter getter;
+
 class ObFakeBackupTabletProvider : public ObIBackupTabletProvider {
 public:
   ObFakeBackupTabletProvider();
@@ -280,6 +284,143 @@ TEST(TestBackupUtils, test_check_macro_block_reuse)
   ASSERT_FALSE(need_skip);
 }
 
+class TestBackupExternalSort : public blocksstable::TestDataFilePrepare
+{
+public:
+  TestBackupExternalSort();
+  virtual ~TestBackupExternalSort() {}
+  int init_tenant_mgr();
+  void destroy_tenant_mgr();
+  int64_t calculate_min_item_count();
+  int generate_items(const int64_t min_count, common::ObIAllocator &allocator, ObVector<ObBackupProviderItem *> &items);
+  virtual void SetUp();
+  virtual void TearDown();
+public:
+  static const int64_t MACRO_BLOCK_SIZE = 2 * 1024 * 1024;
+  static const int64_t MACRO_BLOCK_COUNT = 15 * 1024;
+private:
+  common::ObArenaAllocator allocator_;
+};
+
+TestBackupExternalSort::TestBackupExternalSort()
+  : TestDataFilePrepare(&getter, "TestBackupExternalSort", MACRO_BLOCK_SIZE, MACRO_BLOCK_COUNT),
+    allocator_(ObModIds::TEST)
+{
+}
+
+void TestBackupExternalSort::SetUp()
+{
+  TestDataFilePrepare::SetUp();
+  EXPECT_EQ(OB_SUCCESS, init_tenant_mgr());
+  EXPECT_EQ(OB_SUCCESS, ObTmpFileManager::get_instance().init());
+  static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
+  ObTenantEnv::set_tenant(&tenant_ctx);
+  ObTenantIOManager *io_service = nullptr;
+  EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
+}
+
+void TestBackupExternalSort::TearDown()
+{
+  allocator_.reuse();
+  ObTmpFileManager::get_instance().destroy();
+  TestDataFilePrepare::TearDown();
+  destroy_tenant_mgr();
+}
+
+int TestBackupExternalSort::init_tenant_mgr()
+{
+  int ret = OB_SUCCESS;
+  ObAddr self;
+  self.set_ip_addr("127.0.0.1", 8086);
+  rpc::frame::ObReqTransport req_transport(NULL, NULL);
+  const int64_t ulmt = 128LL << 30;
+  const int64_t llmt = 128LL << 30;
+  ret = getter.add_tenant(OB_SYS_TENANT_ID, ulmt, llmt);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = getter.add_tenant(OB_SERVER_TENANT_ID, ulmt, llmt);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  lib::set_memory_limit(128LL << 32);
+  return ret;
+}
+
+void TestBackupExternalSort::destroy_tenant_mgr()
+{
+}
+
+int64_t TestBackupExternalSort::calculate_min_item_count()
+{
+  ObBackupProviderItem item;
+  const int64_t min_memory_limit = ObExternalSortConstant::MIN_MEMORY_LIMIT;
+  const int64_t item_size = sizeof(item) + item.get_deep_copy_size();
+  int64_t min_count = min_memory_limit / item_size;
+  LOG_INFO("calculate min item count", K(min_memory_limit), K(item_size), K(min_count));
+  return min_count;
+}
+
+int TestBackupExternalSort::generate_items(const int64_t min_count, common::ObIAllocator &allocator, ObVector<ObBackupProviderItem *> &items)
+{
+  int ret = OB_SUCCESS;
+  ObBackupProviderItem *item = NULL;
+  items.reset();
+  if (min_count < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(WARN, "invalid argument", K(ret), K(min_count));
+  } else {
+    void *buf = NULL;
+    ObTabletID tablet_id;
+    for (int64_t i = 0; OB_SUCC(ret) && i < min_count; ++i) {
+      ObBackupProviderItemType item_type = PROVIDER_ITEM_TABLET_META;
+      make_random_tablet_id(tablet_id);
+      if (OB_ISNULL(buf = allocator.alloc(sizeof(ObBackupProviderItem)))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_WARN("failed to allocate memory", K(ret));
+      } else if (OB_ISNULL(item = new (buf) ObBackupProviderItem())) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_WARN("failed to placement new item", K(ret));
+      } else if (OB_FAIL(item->set_with_fake(item_type, tablet_id))) {
+        LOG_WARN("failed to set item", K(ret), K(tablet_id));
+      } else if (OB_FAIL(items.push_back(item))) {
+        LOG_WARN("failed to push back item", K(ret));
+      }
+    }
+  }
+  return ret;
+}
+
+TEST_F(TestBackupExternalSort, test_sort)
+{
+  int ret = OB_SUCCESS;
+  int64_t buf_mem_limit = ObExternalSortConstant::MIN_MEMORY_LIMIT;
+  int64_t file_buf_size = 2 << 20;
+  int64_t expire_timestamp = 0;
+  const uint64_t tenant_id = OB_SYS_TENANT_ID;
+  typedef storage::ObExternalSort<ObBackupProviderItem, ObBackupProviderItemCompare> ExternalSort;
+  ExternalSort external_sort;
+  ObBackupProviderItemCompare backup_item_cmp(ret);
+  ObBackupDataType backup_data_type;
+  backup_data_type.set_major_data_backup();
+  backup_item_cmp.set_backup_data_type(backup_data_type);
+
+  ObArenaAllocator allocator;
+  const int64_t min_count = calculate_min_item_count();
+  ObVector<ObBackupProviderItem *> total_items;
+
+  ret = generate_items(min_count, allocator, total_items);
+  EXPECT_EQ(OB_SUCCESS, ret);
+
+  ret = external_sort.init(buf_mem_limit, file_buf_size, expire_timestamp, tenant_id, &backup_item_cmp);
+  EXPECT_EQ(OB_SUCCESS, ret);
+
+  for (int64_t i = 0; OB_SUCC(ret) && i < total_items.size(); ++i) {
+    const ObBackupProviderItem *item = total_items.at(i);
+    ret = external_sort.add_item(*item);
+    EXPECT_EQ(OB_SUCCESS, ret);
+  }
+
+  ret = external_sort.do_sort(true /*final_merge*/);
+  EXPECT_EQ(OB_SUCCESS, ret);
+}
+
 }  // namespace backup
 }  // namespace oceanbase