[FEAT MERGE] oblogminer[patch from 4.2.3]

Co-authored-by: qiuyg3 <qiuyg3@chinaunicom.cn>

cmake/Pack.cmake

@@ -49,6 +49,7 @@ install(PROGRAMS
   tools/import_time_zone_info.py
   tools/import_srs_data.py
   ${CMAKE_BINARY_DIR}/tools/ob_admin/ob_admin
+  ${CMAKE_BINARY_DIR}/src/logservice/logminer/oblogminer
   tools/ob_admin/io_bench/bench_io.sh
   ${CMAKE_BINARY_DIR}/src/observer/observer
   DESTINATION bin
@@ -369,6 +370,7 @@ if(OB_BUILD_OBADMIN)
     install(PROGRAMS
       ${CMAKE_BINARY_DIR}/tools/ob_admin/ob_admin
       ${CMAKE_BINARY_DIR}/tools/ob_error/src/ob_error
+      ${CMAKE_BINARY_DIR}/src/logservice/logminer/oblogminer
       DESTINATION /usr/bin
       COMPONENT utils
     )

deps/oblib/src/lib/oblog/ob_log.cpp

@@ -1923,6 +1923,11 @@ void ObLogger::issue_dba_error(const int errcode, const char *file, const int li
                 "errcode", errcode, "file", base_file_name, "line_no", line, "info", info_str);
 }
 
+bool ObLogger::is_svr_file_opened()
+{
+  return log_file_[FD_SVR_FILE].is_opened();
+}
+
 }
 }
 

deps/oblib/src/lib/oblog/ob_log.h

@@ -642,6 +642,8 @@ public:
   // issue a ERROR message for each EDIAG log right now
   void issue_dba_error(const int errcode, const char *file, const int line, const char *info_str);
 
+  bool is_svr_file_opened();
+
 private:
   //@brief If version <= 0, return true.
   //If version > 0, return version > level_version_ and if true, update level_version_.

deps/oblib/src/lib/oblog/ob_log_module.h

@@ -75,6 +75,7 @@ DEFINE_LOG_SUB_MOD(MDS)                  // multi data source
 DEFINE_LOG_SUB_MOD(DATA_DICT)            // data_dictionary module
 DEFINE_LOG_SUB_MOD(MVCC)                 // concurrency_control
 DEFINE_LOG_SUB_MOD(WR)                 // workload repository
+DEFINE_LOG_SUB_MOD(LOGMINER)             // logminer
 LOG_MOD_END(ROOT)
 
 //statement of WRS's sub_modules
@@ -459,6 +460,8 @@ LOG_MOD_END(PL)
 #define _MVCC_LOG(level, _fmt_, args...) _OB_MOD_LOG(MVCC, level, _fmt_, ##args)
 #define WR_LOG(level, info_string, args...) OB_MOD_LOG(WR, level, info_string, ##args)
 #define _WR_LOG(level, _fmt_, args...) _OB_MOD_LOG(WR, level, _fmt_, ##args)
+#define LOGMNR_LOG(level, info_string, args...) OB_MOD_LOG(LOGMINER, level, info_string, ##args)
+#define _LOGMNR_LOG(level, _fmt_, args...) _OB_MOD_LOG(LOGMINER, level, _fmt_, ##args)
 
 //dfine ParMod_SubMod_LOG
 #define WRS_CLUSTER_LOG(level, info_string, args...) OB_SUB_MOD_LOG(WRS, CLUSTER, level,        \
@@ -1153,6 +1156,8 @@ LOG_MOD_END(PL)
 #define _OBLOG_DISPATCHER_LOG_RET(level, errcode, args...) { int ret = errcode; _OBLOG_DISPATCHER_LOG(level, ##args); }
 #define OBLOG_SORTER_LOG_RET(level, errcode, args...) { int ret = errcode; OBLOG_SORTER_LOG(level, ##args); }
 #define _OBLOG_SORTER_LOG_RET(level, errcode, args...) { int ret = errcode; _OBLOG_SORTER_LOG(level, ##args); }
+#define LOGMNR_LOG_RET(level, errcode, args...) { int ret = errcode; LOGMNR_LOG(level, ##args); }
+#define _LOGMNR_LOG_RET(level, errcode, args...) { int ret = errcode; _LOGMNR_LOG(level, ##args); }
 #define MDS_LOG_RET(level, errcode, args...) { int ret = errcode; MDS_LOG(level, ##args); }
 #define _MDS_LOG_RET(level, errcode, args...) { int ret = errcode; _MDS_LOG(level, ##args); }
 #define DDLOG_RET(level, errcode, args...){ int ret = errcode; DDLOG(level, ##args); }

deps/oblib/src/lib/oblog/ob_log_module.ipp

@@ -53,6 +53,7 @@ REG_LOG_PAR_MOD(ARCHIVE)
 REG_LOG_PAR_MOD(LOGTOOL)
 REG_LOG_PAR_MOD(DATA_DICT)
 REG_LOG_PAR_MOD(BALANCE)
+REG_LOG_PAR_MOD(LOGMINER)
 
 //regist WRS's sub-modules
 REG_LOG_SUB_MOD(WRS, CLUSTER)

src/logservice/CMakeLists.txt

@@ -238,3 +238,4 @@ ob_set_subtarget(ob_logservice common_util
 
 ob_server_add_target(ob_logservice)
 add_subdirectory(libobcdc)
+add_subdirectory(logminer)

src/logservice/libobcdc/src/ob_log_config.h

@@ -603,7 +603,7 @@ public:
   DEF_CAP(lob_data_storage_memory_limit, OB_CLUSTER_PARAMETER, "1G", "[128M,]", "lob data storage memory limit");
   T_DEF_INT_INFT(lob_data_storage_clean_interval_sec, OB_CLUSTER_PARAMETER, 5, 1,
                  "lob_data_storage clean task nterval in seconds");
-  DEF_TIME(print_mod_memory_usage_interval, OB_CLUSTER_PARAMETER, "1m", "[0s, ]", "print mod memory usage threshold");
+  DEF_TIME(print_mod_memory_usage_interval, OB_CLUSTER_PARAMETER, "1m", "[0s,]", "print mod memory usage interval");
   DEF_CAP(print_mod_memory_usage_threshold, OB_CLUSTER_PARAMETER, "0M", "[0M,]", "print mod memory usage threshold");
   DEF_STR(print_mod_memory_usage_label, OB_CLUSTER_PARAMETER, "|", "mod label for print memmory usage");
   T_DEF_INT_INFT(max_chunk_cache_size, OB_CLUSTER_PARAMETER, 0, 0, "chunkmgr max chunk size");

src/logservice/libobcdc/src/ob_log_instance.cpp

@@ -418,7 +418,9 @@ int ObLogInstance::init_logger_()
     easy_log_level = EASY_LOG_INFO;
     OB_LOGGER.set_max_file_size(MAX_LOG_FILE_SIZE);
     OB_LOGGER.set_max_file_index(max_log_file_count);
-    OB_LOGGER.set_file_name(log_file, disable_redirect_log_, false);
+    if (!OB_LOGGER.is_svr_file_opened()) {
+      OB_LOGGER.set_file_name(log_file, disable_redirect_log_, false);
+    }
     OB_LOGGER.set_log_level("INFO");
     OB_LOGGER.disable_thread_log_level();
     OB_LOGGER.set_enable_log_limit(enable_log_limit);

src/logservice/logminer/CMakeLists.txt

@@ -0,0 +1,44 @@
+ob_set_subtarget(oblogminer_obj_list common
+  ob_log_miner_analysis_writer.cpp
+  ob_log_miner_analyzer_checkpoint.cpp
+  ob_log_miner_analyzer.cpp
+  ob_log_miner_args.cpp
+  ob_log_miner_batch_record.cpp
+  ob_log_miner_batch_record_writer.cpp
+  ob_log_miner_br_converter.cpp
+  ob_log_miner_br_filter.cpp
+  ob_log_miner_br_producer.cpp
+  ob_log_miner_br.cpp
+  ob_log_miner_data_manager.cpp
+  ob_log_miner_file_index.cpp
+  ob_log_miner_file_manager.cpp
+  ob_log_miner_file_meta.cpp
+  ob_log_miner_filter_condition.cpp
+  ob_log_miner_flashback_reader.cpp
+  ob_log_miner_flashback_writer.cpp
+  ob_log_miner_logger.cpp
+  ob_log_miner_mode.cpp
+  ob_log_miner_progress_range.cpp
+  ob_log_miner_record_aggregator.cpp
+  ob_log_miner_record_converter.cpp
+  ob_log_miner_record_file_format.cpp
+  ob_log_miner_record_filter.cpp
+  ob_log_miner_record_parser.cpp
+  ob_log_miner_record_rewriter.cpp
+  ob_log_miner_record.cpp
+  ob_log_miner_resource_collector.cpp
+  ob_log_miner_timezone_getter.cpp
+  ob_log_miner_undo_task.cpp
+  ob_log_miner_utils.cpp
+  ob_log_miner.cpp
+)
+
+add_executable(oblogminer ob_log_miner_main.cpp)
+ob_add_new_object_target(oblogminer_objs oblogminer_obj_list)
+ob_add_new_object_target(oblogminer_obj_dev oblogminer_obj_list)
+# ob_add_new_object_target(oblogminer_main_obj oblogminer_main)
+target_include_directories(oblogminer_obj_dev INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
+# add_library(oblogminer_dev SHARED)
+target_link_libraries(oblogminer_objs PUBLIC obcdc_static)
+target_link_libraries(oblogminer_obj_dev PUBLIC obcdc)
+target_link_libraries(oblogminer PRIVATE oblogminer_objs)

src/logservice/logminer/ob_log_miner.cpp

@@ -0,0 +1,144 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner.h"
+#include "lib/ob_errno.h"
+#include "lib/oblog/ob_log.h"
+#include "ob_log_miner_args.h"
+#include "lib/allocator/ob_malloc.h"
+#include "ob_log_miner_file_manager.h"
+#include "ob_log_miner_utils.h"
+#include "ob_log_miner_logger.h"
+#include "ob_log_miner_record_converter.h"
+#include "ob_log_miner_timezone_getter.h"
+
+namespace oceanbase {
+namespace oblogminer {
+
+ObLogMiner *get_logminer_instance()
+{
+  static ObLogMiner logminer_instance;
+  return &logminer_instance;
+}
+
+ObLogMiner::ObLogMiner():
+    is_inited_(false),
+    is_stopped_(true),
+    mode_(LogMinerMode::UNKNOWN),
+    analyzer_(nullptr),
+    flashbacker_(nullptr),
+    file_manager_(nullptr) { }
+
+ObLogMiner::~ObLogMiner()
+{
+  destroy();
+}
+
+int ObLogMiner::init(const ObLogMinerArgs &args)
+{
+  int ret = common::OB_SUCCESS;
+  if (is_inited_) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("oblogminer was inited, no need to init again", K(is_inited_));
+  } else {
+    mode_ = args.mode_;
+    LOGMINER_LOGGER.set_verbose(args.verbose_);
+    if (OB_FAIL(LOGMINER_TZ.set_timezone(args.timezone_))) {
+      LOG_ERROR("oblogminer logger set timezone failed", K(ret), K(args.timezone_));
+    }
+    if (OB_SUCC(ret)) {
+      if (is_analysis_mode(mode_)) {
+        if (OB_FAIL(init_analyzer_(args))) {
+          LOG_ERROR("failed to initialize analyzer for analysis mode", K(args));
+        }
+      } else {
+        // TODO: flashback mode and restart mode
+        // if (OB_FAIL(init_component<ObLogMinerFileManager>(file_manager_,
+        //     args.flashbacker_args_.recovery_path_, RecordFileFormat::CSV, ObLogMinerFileManager::FileMgrMode::FLASHBACK))) {
+        //   LOG_ERROR("failed to initialize file_manager for flashback mode", K(args));
+        // } else if (OB_FAIL(init_component<ObLogMinerFlashbacker>(flashbacker_,
+        //     args.flashbacker_args_, file_manager_))) {
+        //   LOG_ERROR("flashbacker failed to init", K(args));
+        // } else if (OB_FAIL(file_manager_->write_config(args))) {
+        //   LOG_ERROR("failed to write config", K(args));
+        // }
+        // is_inited_ = true;
+        ret = OB_NOT_SUPPORTED;
+        LOG_ERROR("get not supported mode", K(args));
+      }
+    }
+  }
+  if (OB_SUCC(ret)) {
+    is_inited_ = true;
+    LOG_INFO("ObLogMiner init succeed", K(args));
+    LOGMINER_STDOUT("ObLogMiner init succeed\n");
+  } else {
+    LOG_INFO("ObLogMiner init failed", K(args));
+  }
+  return ret;
+}
+
+int ObLogMiner::init_analyzer_(const ObLogMinerArgs &args)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(init_component<ObLogMinerFileManager>(file_manager_,
+      args.analyzer_args_.output_dst_, args.analyzer_args_.record_format_, ObLogMinerFileManager::FileMgrMode::ANALYZE))) {
+    LOG_ERROR("failed to initialize file_manager for analysis mode", K(args));
+  } else if (OB_FAIL(init_component<ObLogMinerAnalyzer>(analyzer_, args.analyzer_args_, file_manager_))) {
+    LOG_ERROR("analyzer failed to init", K(args));
+  } else if (OB_FAIL(file_manager_->write_config(args))) {
+    LOG_ERROR("failed to write config", K(args));
+  } else if (OB_FAIL(OB_LOGGER.set_mod_log_levels(args.analyzer_args_.log_level_))) {
+    LOG_ERROR("failed to set mod log levels", K(args));
+    LOGMINER_STDOUT("failed to set log level, please check the log level setting\n");
+  }
+  return ret;
+}
+void ObLogMiner::run() {
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMiner hasn't been initiailized yet");
+  } else if (is_analysis_mode(mode_)) {
+    if (OB_FAIL(analyzer_->start())) {
+      LOG_ERROR("analyzer failed to start");
+    } else {
+      analyzer_->wait();
+    }
+  } else if (is_flashback_mode(mode_)) {
+    flashbacker_->run();
+  } else {
+    ret = OB_NOT_SUPPORTED;
+    LOG_ERROR("get not supported mode", K(mode_));
+  }
+}
+
+void ObLogMiner::destroy() {
+  if (IS_INIT) {
+    if (is_analysis_mode(mode_)) {
+      destroy_component<ObLogMinerAnalyzer>(analyzer_);
+      destroy_component<ObLogMinerFileManager>(file_manager_);
+    } else if (is_flashback_mode(mode_)) {
+      destroy_component<ObLogMinerFlashbacker>(flashbacker_);
+      destroy_component<ObLogMinerFileManager>(file_manager_);
+    }
+    is_inited_ = false;
+    mode_ = LogMinerMode::UNKNOWN;
+    LOG_INFO("ObLogMiner destroyed");
+    LOGMINER_STDOUT("ObLogMiner destroyed\n");
+  }
+}
+
+} // namespace oblogminer
+} // namespace oceanbase

src/logservice/logminer/ob_log_miner.h

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_H_
+#define OCEANBASE_LOG_MINER_H_
+
+#include <cstdint>
+#include "ob_log_miner_mode.h"
+#include "ob_log_miner_analyzer.h"
+
+namespace oceanbase
+{
+
+namespace oblogminer
+{
+
+class ILogMinerBRProducer;
+class ILogMinerBRFilter;
+class ILogMinerBRConverter;
+class ILogMinerAnalysisWriter;
+class ILogMinerResourceCollector;
+class ILogMinerDataManager;
+
+class ILogMinerFlashbackReader;
+class ILogMinerRecordParser;
+class ILogMinerRecordFilter;
+class ILogMinerFlashbackWriter;
+class ILogMinerFileManager;
+
+class ObLogMinerArgs;
+class AnalyzerArgs;
+class FlashbackerArgs;
+
+class ObLogMinerFlashbacker {
+public:
+  ObLogMinerFlashbacker() {}
+  virtual ~ObLogMinerFlashbacker() {}
+  int init(const FlashbackerArgs &args, ILogMinerFileManager *file_mgr) { return 0; }
+  void run() {}
+  void stop() {}
+  void destroy() {}
+
+private:
+  bool is_inited_;
+  ILogMinerFlashbackReader    *reader_;
+  ILogMinerRecordParser       *parser_;
+  ILogMinerRecordFilter       *filter_;
+  ILogMinerFlashbackWriter    *writer_;
+};
+
+class ObLogMiner {
+public:
+  static ObLogMiner *get_logminer_instance();
+
+public:
+  ObLogMiner();
+  ~ObLogMiner();
+
+  int init(const ObLogMinerArgs &args);
+
+  // LogMiner would stop automatically if some error occurs or has finished processing logs.
+  // There is no need to implement some interfaces like wait/stop/etc.
+  void run();
+  void destroy();
+private:
+  void set_log_params_();
+  int init_analyzer_(const ObLogMinerArgs &args);
+
+private:
+  bool                        is_inited_;
+  bool                        is_stopped_;
+
+  LogMinerMode                mode_;
+
+  ObLogMinerAnalyzer          *analyzer_;
+  ObLogMinerFlashbacker       *flashbacker_;
+  ILogMinerFileManager        *file_manager_;
+};
+
+} // oblogminer
+
+} // oceanbase
+
+#endif

src/logservice/logminer/ob_log_miner_analysis_writer.cpp

@@ -0,0 +1,150 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_analysis_writer.h"
+#include "ob_log_miner_record_aggregator.h"
+#include "ob_log_miner_batch_record_writer.h"
+#include "ob_log_miner_logger.h"
+
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+ObLogMinerAnalysisWriter::ObLogMinerAnalysisWriter():
+    is_inited_(false),
+    is_stopped_(true),
+    aggregator_(nullptr),
+    batch_writer_(nullptr) { }
+
+ObLogMinerAnalysisWriter::~ObLogMinerAnalysisWriter()
+{
+  destroy();
+}
+
+int ObLogMinerAnalysisWriter::init(const int64_t start_time_us,
+    ILogMinerDataManager *data_manager,
+    ILogMinerResourceCollector *resource_collector,
+    ILogMinerFileManager *file_manager,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("ObLogminerAnalysisWriter has been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data_manager) || OB_ISNULL(resource_collector) || OB_ISNULL(file_manager)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid argument for ObLogMinerAnalysisWriter", K(data_manager),
+        K(resource_collector), K(file_manager));
+  } else if (OB_FAIL(init_component<ObLogMinerBatchRecordWriter>(batch_writer_,
+      data_manager, resource_collector, file_manager, err_handle))) {
+    LOG_ERROR("failed to init ObLogMinerBatchRecordWriter", K(batch_writer_));
+  } else if (OB_FAIL(init_component<ObLogMinerRecordAggregator>(aggregator_,
+      start_time_us, batch_writer_, data_manager, resource_collector, err_handle))) {
+    LOG_ERROR("failed to init ObLogMinerRecordAggregator", K(aggregator_));
+  } else {
+    is_inited_ = true;
+    LOG_INFO("ObLogMinerAnalysisWriter finished to init", KP(data_manager), KP(resource_collector),
+        KP(file_manager), KP(err_handle));
+    LOGMINER_STDOUT_V("ObLogMinerAnalysisWriter finished to init\n");
+  }
+
+  return ret;
+}
+
+int ObLogMinerAnalysisWriter::start()
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerAnalysisWriter hasn't been initialized yet", K(is_inited_));
+  } else if (OB_FAIL(batch_writer_->start())) {
+    LOG_ERROR("batch_writer failed to start");
+  } else if (OB_FAIL(aggregator_->start())) {
+    LOG_ERROR("aggregator failed to start");
+  } else {
+    is_stopped_ = false;
+    LOG_INFO("ObLogMinerAnalysisWriter starts");
+    LOGMINER_STDOUT_V("ObLogMinerAnalysisWriter starts\n");
+  }
+
+  return ret;
+}
+
+void ObLogMinerAnalysisWriter::stop()
+{
+  mark_stop_flag();
+  aggregator_->stop();
+  batch_writer_->stop();
+  LOG_INFO("ObLogMinerAnalysisWriter stopped");
+}
+
+void ObLogMinerAnalysisWriter::wait()
+{
+  aggregator_->wait();
+  batch_writer_->wait();
+  LOG_INFO("ObLogMinerAnalysisWriter finished to wait");
+}
+
+void ObLogMinerAnalysisWriter::destroy()
+{
+  if (IS_INIT) {
+    is_inited_ = false;
+    is_stopped_ = true;
+    destroy_component<ObLogMinerRecordAggregator>(aggregator_);
+    destroy_component<ObLogMinerBatchRecordWriter>(batch_writer_);
+    aggregator_ = nullptr;
+    batch_writer_ = nullptr;
+    LOG_INFO("ObLogMinerAnalysisWriter destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerAnalysisWriter destroyed\n");
+  }
+}
+
+int ObLogMinerAnalysisWriter::push(ObLogMinerRecord *record)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerAnalysisWriter hasn't been initialized yet", K(is_inited_));
+  } else if (OB_FAIL(aggregator_->push(record))) {
+    LOG_ERROR("failed to push record into aggregator", KPC(record));
+  }
+
+  return ret;
+}
+
+int ObLogMinerAnalysisWriter::get_total_task_count(int64_t &task_count)
+{
+  int ret = OB_SUCCESS;
+  int64_t aggregator_record_cnt = 0, batch_writer_record_cnt = 0;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerAnalysisWriter hasn't been initialized yet", K(is_inited_));
+  } else if (OB_FAIL(aggregator_->get_total_record_count(aggregator_record_cnt))) {
+    LOG_ERROR("failed to get aggregator total record count");
+  } else if (OB_FAIL(batch_writer_->get_total_record_count(batch_writer_record_cnt))) {
+    LOG_ERROR("failed to get batch writer record count");
+  } else {
+    task_count = aggregator_record_cnt + batch_writer_record_cnt;
+  }
+
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_analysis_writer.h

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_ANALYSIS_WRITER_H_
+#define OCEANBASE_LOG_MINER_ANALYSIS_WRITER_H_
+
+#include "lib/thread/ob_multi_fixed_queue_thread.h"
+#include "ob_log_miner_error_handler.h"
+namespace oceanbase
+{
+
+namespace oblogminer
+{
+
+class ObLogMinerRecord;
+class ILogMinerFileManager;
+class ILogMinerRecordAggregator;
+class ILogMinerBatchRecordWriter;
+class ILogMinerDataManager;
+class ILogMinerResourceCollector;
+class ILogMinerFileManager;
+
+class ILogMinerAnalysisWriter
+{
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int push(ObLogMinerRecord *logminer_rec) = 0;
+  virtual int get_total_task_count(int64_t &task_count) = 0;
+};
+
+class ObLogMinerAnalysisWriter: public ILogMinerAnalysisWriter
+{
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int push(ObLogMinerRecord *logminer_rec);
+  virtual int get_total_task_count(int64_t &task_count);
+
+public:
+
+  void mark_stop_flag() { ATOMIC_STORE(&is_stopped_, true); }
+  bool has_stopped() const { return ATOMIC_LOAD(&is_stopped_); }
+public:
+  ObLogMinerAnalysisWriter();
+  virtual ~ObLogMinerAnalysisWriter();
+
+  // aggregator and batch_writer is initialized outside analysis writer
+  int init(const int64_t start_time_us,
+      ILogMinerDataManager *data_manager,
+      ILogMinerResourceCollector *resource_collector,
+      ILogMinerFileManager *file_manager,
+      ILogMinerErrorHandler *err_handle);
+private:
+  bool is_inited_;
+  bool is_stopped_;
+  ILogMinerRecordAggregator *aggregator_;
+  ILogMinerBatchRecordWriter *batch_writer_;
+};
+
+}
+
+}
+
+#endif

src/logservice/logminer/ob_log_miner_analyze_schema.h

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifdef MINER_SCHEMA_DEF
+
+MINER_SCHEMA_DEF(TENANT_ID, 0)
+MINER_SCHEMA_DEF(TRANS_ID, 1)
+MINER_SCHEMA_DEF(PRIMARY_KEY, 2)
+// MINER_SCHEMA_DEF(ROW_UNIQUE_ID, 3)
+// MINER_SCHEMA_DEF(SEQ_NO, 4)
+MINER_SCHEMA_DEF(TENANT_NAME, 5)
+MINER_SCHEMA_DEF(DATABASE_NAME, 6)
+MINER_SCHEMA_DEF(TABLE_NAME, 7)
+MINER_SCHEMA_DEF(OPERATION, 8)
+MINER_SCHEMA_DEF(OPERATION_CODE, 9)
+MINER_SCHEMA_DEF(COMMIT_SCN, 10)
+MINER_SCHEMA_DEF(COMMIT_TIMESTAMP, 11)
+MINER_SCHEMA_DEF(SQL_REDO, 12)
+MINER_SCHEMA_DEF(SQL_UNDO, 13)
+MINER_SCHEMA_DEF(ORG_CLUSTER_ID, 14)
+
+#endif

src/logservice/logminer/ob_log_miner_analyzer.cpp

@@ -0,0 +1,239 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_analyzer.h"
+#include "ob_log_miner_utils.h"
+#include "ob_log_miner_br_producer.h"
+#include "ob_log_miner_br_filter.h"
+#include "ob_log_miner_br_converter.h"
+#include "ob_log_miner_args.h"
+#include "ob_log_miner_analysis_writer.h"
+#include "ob_log_miner_data_manager.h"
+#include "ob_log_miner_resource_collector.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+ObLogMinerAnalyzer::ObLogMinerAnalyzer():
+    lib::ThreadPool(1),
+    is_inited_(false),
+    is_stopped_(false),
+    start_time_(-1),
+    producer_(nullptr),
+    data_filter_(nullptr),
+    data_converter_(nullptr),
+    writer_(nullptr),
+    data_manager_(nullptr),
+    resource_collector_(nullptr),
+    file_manager_(nullptr) { }
+
+ObLogMinerAnalyzer::~ObLogMinerAnalyzer()
+{
+  destroy();
+}
+
+int ObLogMinerAnalyzer::init(const AnalyzerArgs &args, ILogMinerFileManager *file_mgr)
+{
+  int ret = OB_SUCCESS;
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("analyzer has been initialized", K(is_inited_));
+  } else {
+    LOGMINER_STDOUT("ObLogMinerAnalyzer init...\n");
+    file_manager_ = file_mgr;
+    if (OB_FAIL(lib::ThreadPool::init())) {
+      LOG_ERROR("analyzer thread pool failed to init");
+    } else if (OB_FAIL(init_component<ObLogMinerDataManager>(data_manager_,
+        LogMinerMode::ANALYSIS, args.record_format_, args.start_time_us_, args.end_time_us_, this))) {
+      LOG_ERROR("failed to init ObLogMinerDataManager", K(args));
+    } else if (OB_FAIL(init_component<ObLogMinerResourceCollector>(resource_collector_,
+        data_manager_, this))) {
+      LOG_ERROR("failed to init ObLogMinerResourceCollector", K(args));
+    } else if (OB_FAIL(init_component<ObLogMinerAnalysisWriter>(writer_,
+      args.start_time_us_, data_manager_, resource_collector_, file_manager_, this))) {
+      LOG_ERROR("failed to init ObLogMinerAnalysisWriter", K(args));
+    } else if (OB_FAIL(init_component<ObLogMinerBRConverter>(data_converter_,
+        data_manager_, writer_, resource_collector_, this))) {
+      LOG_ERROR("failed to init ObLogMinerBRConverter", K(args));
+    } else if (OB_FAIL(init_component<ObLogMinerBRFilter>(data_filter_,
+        args.column_cond_, args.operations_, data_manager_, resource_collector_, data_converter_, this))) {
+      LOG_ERROR("failed to init ObLogMinerBRFilter", K(args));
+    } else if (OB_FAIL(init_component<ObLogMinerBRProducer>(producer_, args, data_filter_, this))) {
+      LOG_ERROR("failed to init ObLogMinerBRProducer", K(args));
+    } else {
+      is_inited_ = true;
+      LOG_INFO("ObLogMinerAnalyzer finished to init", K(args), KP(file_mgr));
+      LOGMINER_STDOUT("ObLogMinerAnalyzer finished to init\n");
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerAnalyzer::start()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("analyzer hasn't been initialized", K(is_inited_));
+  } else if (OB_FAIL(data_manager_->start())) {
+    LOG_ERROR("data_manager failed to start");
+  } else if (OB_FAIL(resource_collector_->start())) {
+    LOG_ERROR("resource_collector failed to start");
+  } else if (OB_FAIL(writer_->start())) {
+    LOG_ERROR("analyzer_writer failed to start");
+  } else if (OB_FAIL(data_converter_->start())) {
+    LOG_ERROR("data_converter failed to start");
+  } else if (OB_FAIL(data_filter_->start())) {
+    LOG_ERROR("data_filter failed to start");
+  } else if (OB_FAIL(producer_->start())) {
+    LOG_ERROR("data_producer failed to start");
+  } else if (OB_FAIL(lib::ThreadPool::start())) {
+    LOG_ERROR("analyzer thread failed to start");
+  } else {
+    is_stopped_ = false;
+    start_time_ = ObClockGenerator::getClock();
+    LOG_INFO("ObLogMinerAnalyzer starts");
+    LOGMINER_STDOUT("ObLogMinerAnalyzer starts\n");
+  }
+
+  return ret;
+}
+
+void ObLogMinerAnalyzer::stop()
+{
+  int ret = OB_SUCCESS;
+  double interval_sec = (double)(ObClockGenerator::getClock() - start_time_) / (double)1000000;
+  LOGMINER_STDOUT("ObLogMinerAnalyzer took %.2lfs to process\n", interval_sec);
+  LOGMINER_STDOUT("ObLogMinerAnalyzer exit...\n");
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("analyzer hasn't been initialized");
+  } else {
+    lib::Threads::stop();
+    data_filter_->stop();
+    data_converter_->stop();
+    writer_->stop();
+    resource_collector_->stop();
+    data_manager_->stop();
+    // resource collector depends on producer
+    producer_->stop();
+    ATOMIC_STORE(&is_stopped_, true);
+    stop_cond_.signal();
+    LOG_INFO("ObLogMinerAnalyzer stopped");
+    LOGMINER_STDOUT_V("ObLogMinerAnalyzer stopped\n");
+  }
+}
+
+void ObLogMinerAnalyzer::wait()
+{
+  if (IS_INIT) {
+    const int64_t wait_time_interval = 10L * 1000 * 1000;
+    while (! ATOMIC_LOAD(&is_stopped_)) {
+      stop_cond_.timedwait(wait_time_interval);
+    }
+    lib::Threads::wait();
+    writer_->wait();
+    data_converter_->wait();
+    data_filter_->wait();
+    data_manager_->wait();
+    resource_collector_->wait();
+    producer_->wait();
+  }
+  LOG_INFO("ObLogMinerAnalyzer finished to wait");
+  LOGMINER_STDOUT_V("ObLogMinerAnalyzer finished to wait\n");
+}
+
+void ObLogMinerAnalyzer::destroy()
+{
+  if (IS_INIT) {
+    is_inited_ = false;
+    start_time_ = -1;
+    lib::Threads::destroy();
+    destroy_component<ObLogMinerBRProducer>(producer_);
+    destroy_component<ObLogMinerBRFilter>(data_filter_);
+    destroy_component<ObLogMinerBRConverter>(data_converter_);
+    destroy_component<ObLogMinerAnalysisWriter>(writer_);
+    destroy_component<ObLogMinerResourceCollector>(resource_collector_);
+    destroy_component<ObLogMinerDataManager>(data_manager_);
+    file_manager_ = nullptr;
+    LOG_INFO("ObLogMinerAnalyzer destroyed");
+    LOGMINER_STDOUT("ObLogMinerAnalyzer destroyed\n");
+  }
+}
+
+void ObLogMinerAnalyzer::handle_error(int err_code, const char *fmt, ...)
+{
+  int ret = OB_SUCCESS;
+  constexpr int64_t err_msg_len = 1024;
+  char err_msg[err_msg_len] = {0};
+  static int stop_once = 0;
+
+  if (IS_INIT) {
+    va_list arg;
+    ret = err_code;
+    va_start(arg, fmt);
+    vsnprintf(err_msg, err_msg_len, fmt, arg);
+    if (ret != OB_SUCCESS) {
+      LOG_ERROR(err_msg);
+      LOGMINER_STDOUT(err_msg);
+      LOGMINER_STDOUT("please check log[%s] for more detail\n", ObLogMinerArgs::LOGMINER_LOG_FILE);
+    } else {
+      LOG_INFO(err_msg);
+    }
+    va_end(arg);
+    if (ATOMIC_BCAS(&stop_once, 0, 1)) {
+      stop();
+    }
+  }
+}
+
+void ObLogMinerAnalyzer::run1()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("analyzer hasn't been initialized");
+  } else {
+    const int64_t STAT_INTERVAL = 10L * 1000 * 1000; // 10s
+    const int64_t LOOP_INTERVAL = 1L * 1000 * 1000; // 1s
+    while (!has_set_stop()) {
+      if (REACH_TIME_INTERVAL(STAT_INTERVAL)) {
+        int64_t filter_task_cnt = 0;
+        int64_t converter_task_cnt = 0;
+        int64_t writer_task_cnt = 0;
+        int64_t rc_task_cnt = 0;
+        if (OB_FAIL(data_filter_->get_total_task_count(filter_task_cnt))) {
+          LOG_ERROR("failed to get task count in data_filter");
+        } else if (OB_FAIL(data_converter_->get_total_task_count(converter_task_cnt))) {
+          LOG_ERROR("failed to get task count in data_converter");
+        } else if (OB_FAIL(writer_->get_total_task_count(writer_task_cnt))) {
+          LOG_ERROR("failed to get task count in writer");
+        } else if (OB_FAIL(resource_collector_->get_total_task_count(rc_task_cnt))) {
+          LOG_ERROR("failed to get task count in resource collector");
+        } else {
+          LOG_INFO("[ANALYZER][TASK_STAT]", "Filter", filter_task_cnt, "Converter", converter_task_cnt,
+              "Writer", writer_task_cnt, "ResourceCollector", rc_task_cnt);
+        }
+      }
+      ob_usleep(LOOP_INTERVAL);
+    }
+  }
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_analyzer.h

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_ANALYZER_H_
+#define OCEANBASE_LOG_MINER_ANALYZER_H_
+
+#include "ob_log_miner_error_handler.h"
+#include "ob_queue_thread.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerBRProducer;
+class ILogMinerBRFilter;
+class ILogMinerBRConverter;
+class ILogMinerAnalysisWriter;
+class ILogMinerResourceCollector;
+class ILogMinerDataManager;
+class ILogMinerFileManager;
+class AnalyzerArgs;
+
+class ObLogMinerAnalyzer: public ILogMinerErrorHandler, lib::ThreadPool {
+public:
+  ObLogMinerAnalyzer();
+  virtual ~ObLogMinerAnalyzer();
+  int init(const AnalyzerArgs &args, ILogMinerFileManager *file_mgr);
+  int start();
+  void stop();
+  void wait();
+  void destroy();
+public:
+  void handle_error(int err_code, const char *fmt, ...);
+
+public:
+  virtual void run1();
+
+private:
+  bool is_inited_;
+  bool is_stopped_;
+  int64_t start_time_;
+  common::ObCond stop_cond_;
+  ILogMinerBRProducer         *producer_;
+  ILogMinerBRFilter           *data_filter_;
+  ILogMinerBRConverter        *data_converter_;
+  ILogMinerAnalysisWriter     *writer_;
+  ILogMinerDataManager        *data_manager_;
+  ILogMinerResourceCollector  *resource_collector_;
+  ILogMinerFileManager        *file_manager_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_analyzer_checkpoint.cpp

@@ -0,0 +1,68 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_analyzer_checkpoint.h"
+#include "ob_log_miner_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+////////////////////////////// ObLogMinerCheckpoint //////////////////////////////
+const char * ObLogMinerCheckpoint::progress_key_str = "PROGRESS";
+const char * ObLogMinerCheckpoint::cur_file_id_key_str = "CUR_FILE_ID";
+const char * ObLogMinerCheckpoint::max_file_id_key_str = "MAX_FILE_ID";
+
+int ObLogMinerCheckpoint::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n%s=%ld\n%s=%ld\n",
+      progress_key_str, progress_, cur_file_id_key_str, cur_file_id_, max_file_id_key_str, max_file_id_))) {
+    LOG_ERROR("failed to fill formatted checkpoint into buffer", K(buf_len), K(pos), KPC(this));
+  }
+  return ret;
+}
+
+int ObLogMinerCheckpoint::deserialize(const char *buf, const int64_t data_len, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_FAIL(parse_line(progress_key_str, buf, data_len, pos, progress_))) {
+    LOG_ERROR("parse progress failed", K(progress_key_str), K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(cur_file_id_key_str, buf, data_len, pos, cur_file_id_))) {
+    LOG_ERROR("parse cur_file_id failed", K(cur_file_id_key_str), K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(max_file_id_key_str, buf, data_len, pos, max_file_id_))) {
+    LOG_ERROR("parse max_file_id failed", K(max_file_id_key_str), K(data_len), K(pos));
+  }
+
+  return ret;
+}
+
+int64_t ObLogMinerCheckpoint::get_serialize_size() const
+{
+  int64_t len = 0;
+  const int64_t digit_max_len = 30;
+  char digit[digit_max_len] = {0};
+  len += snprintf(digit, digit_max_len, "%ld", progress_);
+  len += snprintf(digit, digit_max_len, "%ld", cur_file_id_);
+  len += snprintf(digit, digit_max_len, "%ld", max_file_id_);
+  len += strlen(progress_key_str) + 2 + strlen(cur_file_id_key_str) + 2 + strlen(max_file_id_key_str) + 2;
+  return len;
+}
+
+
+
+}
+}

src/logservice/logminer/ob_log_miner_analyzer_checkpoint.h

@@ -0,0 +1,70 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_ANALYZER_CHECKPOINT_H_
+#define OCEANBASE_LOG_MINER_ANALYZER_CHECKPOINT_H_
+
+#include "lib/ob_define.h"
+#include "lib/utility/ob_print_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+struct ObLogMinerCheckpoint
+{
+public:
+  ObLogMinerCheckpoint() { reset(); }
+  ~ObLogMinerCheckpoint() { reset(); }
+
+  void reset() {
+    progress_ = OB_INVALID_TIMESTAMP;
+    max_file_id_ = -1;
+    cur_file_id_ = -1;
+  }
+
+  ObLogMinerCheckpoint &operator=(const ObLogMinerCheckpoint &that)
+  {
+    progress_ = that.progress_;
+    max_file_id_ = that.max_file_id_;
+    cur_file_id_ = that.cur_file_id_;
+    return *this;
+  }
+
+  bool operator==(const ObLogMinerCheckpoint &that) const
+  {
+    return progress_ == that.progress_ && cur_file_id_ == that.cur_file_id_ && max_file_id_ == that.max_file_id_;
+  }
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  TO_STRING_KV(
+    K(progress_),
+    K(max_file_id_),
+    K(cur_file_id_)
+  );
+private:
+  static const char *progress_key_str;
+  static const char *cur_file_id_key_str;
+  static const char *max_file_id_key_str;
+
+public:
+  int64_t progress_;
+  int64_t max_file_id_;
+  int64_t cur_file_id_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_args.cpp

@@ -0,0 +1,920 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "lib/container/ob_array_wrap.h"
+#include "ob_log_miner_utils.h"                       // logminer_str2ll
+#include "ob_log_miner_args.h"                        // ObLogMinerCmdArgs/ObLogMinerArgs/AnalyzerArgs/FlashbackerArgs
+#include "ob_log_miner_logger.h"                      // LOGMINER_LOGGER
+#include "lib/ob_define.h"                            // OB_SUCCESS
+#include "lib/oblog/ob_log.h"                         // LOG_XX
+#include "lib/timezone/ob_time_convert.h"             // ObTimeConverter
+#include "lib/timezone/ob_timezone_info.h"            // ObTimeZoneInfo
+#include <getopt.h>                                   // getopt_long
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+/////////////////////////////////////// ObLogMinerCmdArgs ///////////////////////////////////////
+
+void ObLogMinerCmdArgs::print_usage(const char *prog_name)
+{
+  fprintf(stderr,
+  "OceanBase LogMiner(ObLogMiner) is a tool that converts CLOG in OceanBase to\n"
+  "a human-readable format.\n"
+  "The following are examples demonstrating how to use:\n"
+  "%s -c \"ip:port\" -u \"user@tenant\" -p \"***\" -s \"2023-01-01 10:00:00\" -o \"file:///data/output/\"\n"
+  "%s -a \"file:///data/archive/\" -s \"2023-01-01 10:00:00\" -o \"file:///data/output/\"\n"
+  "Options are list below:\n"
+  "   -m, --mode             Specifies the working mode of ObLogMiner, which currently\n"
+  "                          only supports \"analysis\" mode. The default is \"analysis\".\n"
+  "   -c, --cluster-addr     A list of IPs and ports separated by '|' (e.g., ip1:port1|ip2:port2),\n"
+  "                          where the IP is the observer's IP and the port is the observer's SQL port.\n"
+  "   -u, --user-name        The tenant's username, including the tenant name(e.g., user@tenant).\n"
+  "   -p, --password         The password corresponding to the user-name.\n"
+  "   -a, --archive-dest     The location for the tenant's archive log(e.g., file:///data/xxx/xx).\n"
+  "   -l, --table-list       Specifies the table list for analysis, corresponding to\n"
+  "                          the `tb_white_list` parameter in OBCDC.\n"
+  "   -C, --column-cond      Specifies the column filter for analysis.\n"
+  "   -s, --start-time       Specifies the start time in microsecond timestamp or datetime format.\n"
+  "   -e, --end-time         Specifies the end time in microsecond timestamp or datetime format.\n"
+  "   -O, --operations       Specifies the type of operations, options including insert,\n"
+  "                          update, and delete. Operations can be combined using '|'.\n"
+  "                          The default is \"insert|update|delete\".\n"
+  "   -o, --output           The location for the analysis results.\n"
+  "   -h, --help             Displays the help message.\n"
+  "   -v, --verbose          Determines whether to output detailed logs to the command line.\n"
+  "   -z, --timezone         Specifies the timezone, default is \"+8:00\".\n"
+  "   -f, --record_format    Specifies the record format, default is \"CSV\".\n"
+  "                          Options including CSV, REDO_ONLY, UNDO_ONLY, JSON.\n"
+  "   -L  --log_level        Specifies the log level, default is \"ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN\".\n"
+  "\n",
+  prog_name, prog_name
+  );
+}
+
+void ObLogMinerCmdArgs::reset()
+{
+  mode_ = LogMinerMode::UNKNOWN;
+  cluster_addr_ = nullptr;
+  user_name_ = nullptr;
+  password_ = nullptr;
+  archive_dest_ = nullptr;
+  table_list_ = nullptr;
+  column_cond_ = nullptr;
+  operations_ = nullptr;
+  output_ = nullptr;
+  recovery_path_ = nullptr;
+  target_table_ = nullptr;
+  source_table_ = nullptr;
+  start_time_ = nullptr;
+  end_time_ = nullptr;
+  log_level_ = nullptr;
+  timezone_ = nullptr;
+  record_format_ = nullptr;
+  verbose_ = false;
+  print_usage_ = false;
+}
+
+int ObLogMinerCmdArgs::init(int argc, char **argv)
+{
+  int ret = OB_SUCCESS;
+  int opt = -1;
+  const char *short_opts = "m:c:u:p:a:l:C:s:e:O:o:r:t:S:hL:z:f:v";
+  const struct option long_opts[] = {
+    {"mode", 1, NULL, 'm'},
+    {"cluster-addr", 1, NULL, 'c'},
+    {"user-name", 1, NULL, 'u'},
+    {"password", 1, NULL, 'p'},
+    {"archive-dest", 1, NULL, 'a'},
+    {"table-list", 1, NULL, 'l'},
+    {"column-cond", 1, NULL, 'C'},
+    {"start-time", 1, NULL, 's'},
+    {"end-time", 1, NULL, 'e'},
+    {"operations", 1, NULL, 'O'},
+    {"output", 1, NULL, 'o'},
+    {"recovery-path", 1, NULL, 'r'},
+    {"target-table", 1, NULL, 't'},
+    {"source-table", 1, NULL, 'S'},
+    {"help", 0, NULL, 'h'},
+    {"log_level", 1, NULL, 'L'},
+    {"timezone", 1, NULL, 'z'},
+    {"record_format", 1, NULL, 'f'},
+    {"verbose", 0, NULL, 'v'},
+    {0, 0, 0, 0},
+  };
+
+  if (argc <= 1) {
+    // print usage, just as -h
+    print_usage(argv[0]);
+    ret = OB_IN_STOP_STATE;
+  }
+  // set logminer default mode.
+  if (OB_SUCC(ret)) {
+    mode_ = LogMinerMode::ANALYSIS;
+  }
+
+  while (OB_SUCC(ret) && (-1 != (opt = getopt_long(argc, argv, short_opts, long_opts, NULL)))) {
+    _LOG_TRACE("get opt %d, val is \"%s\"", opt, optarg);
+    switch (opt) {
+      case 'm': {
+        mode_ = get_logminer_mode(optarg);
+        break;
+      }
+
+      case 'c': {
+        cluster_addr_ = optarg;
+        break;
+      }
+
+      case 'u': {
+        user_name_ = optarg;
+        break;
+      }
+
+      case 'p': {
+        password_ = optarg;
+        break;
+      }
+
+      case 'a': {
+        archive_dest_ = optarg;
+        break;
+      }
+
+      case 'l': {
+        table_list_ = optarg;
+        break;
+      }
+
+      case 'C': {
+        column_cond_ = optarg;
+        break;
+      }
+
+      case 's': {
+        start_time_ = optarg;
+        break;
+      }
+
+      case 'e': {
+        end_time_ = optarg;
+        break;
+      }
+
+      case 'o': {
+        output_ = optarg;
+        break;
+      }
+
+      case 'O': {
+        operations_ = optarg;
+        break;
+      }
+
+      case 'r': {
+        recovery_path_ = optarg;
+        break;
+      }
+
+      case 't': {
+        target_table_ = optarg;
+        break;
+      }
+
+      case 'S': {
+        source_table_ = optarg;
+        break;
+      }
+
+      case 'h': {
+        print_usage_ = true;
+        break;
+      }
+
+      case 'L': {
+        log_level_ = optarg;
+        break;
+      }
+
+      case 'z': {
+        timezone_ = optarg;
+        break;
+      }
+
+      case 'f': {
+        record_format_ = optarg;
+        break;
+      }
+
+      case 'v': {
+        verbose_ = true;
+        break;
+      }
+
+      default: {
+        ret = OB_INVALID_ARGUMENT;
+        break;
+      }
+    }
+  }
+  LOG_INFO("parse cmd args finished", KPC(this));
+  if (OB_SUCC(ret) && OB_FAIL(validate_())) {
+    LOG_ERROR("logminer command arguments are invalid", KPC(this));
+    LOGMINER_STDOUT("logminer command arguments are invalid\n");
+  }
+
+  return ret;
+}
+
+int ObLogMinerCmdArgs::validate_() const
+{
+  int ret = OB_SUCCESS;
+  if (print_usage_) {
+    // do nothing
+  } else if (! is_logminer_mode_valid(mode_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("logminer mode is invalid", K(mode_));
+    LOGMINER_STDOUT("logminer mode is invalid\n");
+  } else if (is_analysis_mode(mode_)) {
+    const bool server_args_set = (nullptr != cluster_addr_) || (nullptr != user_name_) || (nullptr != password_);
+    const bool server_args_complete = (nullptr != cluster_addr_) && (nullptr != user_name_) && (nullptr != password_);
+    const bool archive_args_set = (nullptr != archive_dest_);
+    const bool is_other_required_args_set = (nullptr != start_time_) && (nullptr != output_);
+    const bool is_flashback_spec_args_set = (nullptr != recovery_path_) || (nullptr != target_table_) ||
+                                            (nullptr != source_table_);
+    if (archive_args_set && server_args_set) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("server_args and archive_args are both set", K(server_args_set), K(archive_args_set));
+      LOGMINER_STDOUT("(cluster_addr, user_name, password) and (archive_dest) are both specified\n");
+    } else if (!server_args_complete && !archive_args_set) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("server_args and archive_args are both empty or incomplete", K(server_args_set), K(archive_args_set));
+      LOGMINER_STDOUT("(cluster_addr, user_name, password) and (archive_dest) are both empty or incomplete\n");
+    } else if (! is_other_required_args_set) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("start_time or output must be set in analysis mode");
+      LOGMINER_STDOUT("start_time or output are empty\n");
+    } else if (is_flashback_spec_args_set) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("some flashback specified arguments are set in analysis mode",
+          KCSTRING(target_table_), KCSTRING(source_table_));
+      LOGMINER_STDOUT("some flashback specified arguments are set in analysis mode\n");
+    }
+  } else if (is_flashback_mode(mode_)) {
+    const bool server_args_set = (nullptr != cluster_addr_) && (nullptr != user_name_) && (nullptr != password_);
+    const bool is_other_required_args_set = (nullptr != recovery_path_) &&
+                                            (nullptr != target_table_) && (nullptr != source_table_) &&
+                                            (nullptr != start_time_) && (nullptr != end_time_) &&
+                                            (nullptr != output_);
+    const bool is_analysis_spec_args_set = (nullptr != table_list_) || (nullptr != operations_) ||
+                                           (nullptr != archive_dest_);
+
+    if (!server_args_set || !is_other_required_args_set) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("some of the required arguments are not set in flashback mode", KCSTRING(cluster_addr_), KCSTRING(user_name_),
+          KCSTRING(target_table_), KCSTRING(source_table_), KCSTRING(start_time_), KCSTRING(end_time_));
+    } else if (is_analysis_spec_args_set) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("some analysis specified arguments are set in flashback mode", KCSTRING(table_list_), KCSTRING(operations_));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("logminer mode is invalid", K(mode_));
+    LOGMINER_STDOUT("logminer mode is invalid\n");
+  }
+
+  return ret;
+}
+
+/////////////////////////////////////// AnalyzerArgs ///////////////////////////////////////
+
+const char *AnalyzerArgs::CLUSTER_ADDR_KEY = "cluster_addr";
+const char *AnalyzerArgs::USER_NAME_KEY = "user_name";
+const char *AnalyzerArgs::PASSWORD_KEY = "password";
+const char *AnalyzerArgs::ARCHIVE_DEST_KEY = "archive_dest";
+const char *AnalyzerArgs::TABLE_LIST_KEY = "table_list";
+const char *AnalyzerArgs::COLUMN_COND_KEY = "column_cond";
+const char *AnalyzerArgs::OPERATIONS_KEY = "operations";
+const char *AnalyzerArgs::OUTPUT_DST_KEY = "output_dest";
+const char *AnalyzerArgs::LOG_LEVEL_KEY = "log_level";
+const char *AnalyzerArgs::START_TIME_US_KEY = "start_time_us";
+const char *AnalyzerArgs::END_TIME_US_KEY = "end_time_us";
+const char *AnalyzerArgs::TIMEZONE_KEY = "timezone";
+const char *AnalyzerArgs::RECORD_FORMAT_KEY = "record_format";
+void AnalyzerArgs::reset()
+{
+  cluster_addr_ = nullptr;
+  user_name_ = nullptr;
+  password_ = nullptr;
+  archive_dest_ = nullptr;
+  table_list_ = nullptr;
+  column_cond_ = nullptr;
+  operations_ = nullptr;
+  output_dst_ = nullptr;
+  log_level_ = nullptr;
+  start_time_us_ = OB_INVALID_TIMESTAMP;
+  end_time_us_ = OB_INVALID_TIMESTAMP;
+  timezone_ = nullptr;
+  record_format_ = RecordFileFormat::INVALID;
+  alloc_.reset();
+}
+
+int AnalyzerArgs::init(const ObLogMinerCmdArgs &args)
+{
+  int ret = OB_SUCCESS;
+  if (! is_analysis_mode(args.mode_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("incorrect cmd args to init analyzer args", K(args));
+  } else {
+    const char *operations = nullptr == args.operations_ ? ObLogMinerArgs::DEFAULT_LOGMNR_OPERATIONS : args.operations_;
+    const char *timezone = nullptr == args.timezone_ ? ObLogMinerArgs::DEFAULT_LOGMNR_TIMEZONE : args.timezone_;
+    const char *table_list = nullptr == args.table_list_ ? ObLogMinerArgs::DEFAULT_LOGMNR_TABLE_LIST : args.table_list_;
+    const char *record_format_str = nullptr == args.record_format_ ? ObLogMinerArgs::DEFAULT_LOGMNR_FORMAT : args.record_format_;
+    ObTimeZoneInfo tz_info;
+    ObTimeConvertCtx cvrt_ctx(&tz_info, true);
+    if (OB_FAIL(deep_copy_cstring(alloc_, args.cluster_addr_, cluster_addr_))) {
+      LOG_ERROR("failed to deep copy cluster addr", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, args.user_name_, user_name_))) {
+      LOG_ERROR("failed to deep copy user name", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, args.password_, password_))) {
+      LOG_ERROR("failed to deep copy password", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, args.archive_dest_, archive_dest_))) {
+      LOG_ERROR("failed to deep copy archive dest", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, table_list, table_list_))) {
+      LOG_ERROR("failed to deep copy table list", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, args.column_cond_, column_cond_))) {
+      LOG_ERROR("failed to deep copy column cond", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, operations, operations_))) {
+      LOG_ERROR("failed to deep copy operations", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, args.output_, output_dst_))) {
+      LOG_ERROR("failed to deep copy output dest", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, args.log_level_, log_level_))) {
+      LOG_ERROR("failed to deep copy log level", K(args));
+    } else if (OB_FAIL(deep_copy_cstring(alloc_, timezone, timezone_))) {
+      LOG_ERROR("failed to deep copy timezone", K(args));
+    }
+
+    if (OB_SUCC(ret)) {
+      if (OB_FAIL(tz_info.set_timezone(ObString(timezone_)))) {
+        LOG_ERROR("parse timezone failed", K(ret), KCSTRING(timezone_));
+        LOGMINER_STDOUT("parse timezone failed\n");
+      } else {
+        record_format_ = get_record_file_format(record_format_str);
+        if (RecordFileFormat::INVALID == record_format_) {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_ERROR("parse record_format failed", K(ret), KCSTRING(record_format_str));
+          LOGMINER_STDOUT("parse record_format failed\n");
+        }
+      }
+    }
+    if (OB_SUCC(ret)) {
+      if (is_number(args.start_time_)) {
+        if (OB_FAIL(logminer_str2ll(args.start_time_, start_time_us_))) {
+          LOG_ERROR("parse start time for AnalyzerArgs failed", K(ret), K(args),
+              K(start_time_us_), K(errno), K(strerror(errno)));
+        }
+      } else if (OB_FAIL(ObTimeConverter::str_to_datetime(ObString(args.start_time_), cvrt_ctx, start_time_us_))) {
+        LOG_ERROR("parse start time for AnalyzerArgs failed", K(ret), K(args), K(start_time_us_));
+      }
+      if (OB_FAIL(ret)) {
+        LOGMINER_STDOUT("parse start time for AnalyzerArgs failed\n");
+      }
+    }
+    if (OB_SUCC(ret)) {
+      if (nullptr == args.end_time_) {
+        end_time_us_ = ObTimeUtility::current_time();
+      } else if (is_number(args.end_time_)) {
+        if (OB_FAIL(logminer_str2ll(args.end_time_, end_time_us_))) {
+          LOG_ERROR("parse end time for AnalyzerArgs failed", K(ret), K(args), K(end_time_us_), K(errno),
+              K(strerror(errno)));
+        }
+      } else if (OB_FAIL(ObTimeConverter::str_to_datetime(ObString(args.end_time_), cvrt_ctx, end_time_us_))) {
+        LOG_ERROR("parse end time for AnalyzerArgs failed", K(ret), K(args), K(end_time_us_));
+      }
+      if (OB_FAIL(ret)) {
+        LOGMINER_STDOUT("parse end time for AnalyzerArgs failed\n");
+      }
+    }
+
+    if (OB_SUCC(ret)) {
+      if (OB_FAIL(validate())) {
+        LOG_ERROR("validate AnalyzerArgs failed after initialize all arguments", K(args), KPC(this));
+      } else {
+        // succ
+        LOG_INFO("finished to init analyzer_args", KPC(this));
+      }
+    }
+  }
+  return ret;
+}
+
+int AnalyzerArgs::validate() const
+{
+  int ret = OB_SUCCESS;
+  bool is_server_avail = (cluster_addr_ != nullptr) && (user_name_ != nullptr) && (password_ != nullptr);
+  bool is_server_arg_set = (cluster_addr_ != nullptr) || (user_name_ != nullptr) || (password_ != nullptr);
+  bool is_archive_avail = (archive_dest_ != nullptr);
+  if ((is_archive_avail && is_server_arg_set) || (!is_archive_avail && !is_server_avail)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("(cluster_addr, user_name, password) and (archive_dest) are both"
+        " specified or not completely specified", K(is_server_avail), K(is_archive_avail));
+    LOGMINER_STDOUT("(cluster_addr, user_name, password) and (archive_dest) are both"
+        " specified or not completely specified\n");
+  } else if (OB_INVALID_TIMESTAMP == start_time_us_ ||
+             OB_INVALID_TIMESTAMP == end_time_us_ ||
+             start_time_us_ >= end_time_us_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("start_time or end_time is not valid or start_time exceeds end_time",
+        K(start_time_us_), K(end_time_us_));
+    LOGMINER_STDOUT("start_time or end_time is not valid or start_time exceeds end_time\n");
+  } else if (nullptr == output_dst_) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("output_dst is null which is not expected", KPC(this));
+    LOGMINER_STDOUT("output_dst should be specified\n");
+  } else {
+    const char *user_name_delim = "@#";
+    char *user_name_buf = nullptr;
+    char *strtok_ptr = nullptr;
+    char *user = nullptr;
+    char *tenant = nullptr;
+    int user_name_size = 0;
+    if (is_server_avail) {
+      // only support user tenant
+      user_name_size = strlen(user_name_) + 1;
+      if (OB_ISNULL(user_name_buf = reinterpret_cast<char*>(ob_malloc(user_name_size, "LogMnrUsrBuf")))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_ERROR("err alloc string buffer", K(ret), K(user_name_size));
+      } else {
+        MEMCPY(user_name_buf, user_name_, user_name_size-1);
+        user_name_buf[user_name_size-1] = '\0';
+      }
+      user = STRTOK_R(user_name_buf, user_name_delim, &strtok_ptr);
+      tenant = STRTOK_R(nullptr, user_name_delim, &strtok_ptr);
+      if (nullptr == tenant || 0 == strcmp(tenant, OB_SYS_TENANT_NAME)) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("oblogminer only support user tenant(e.g., user@tenant).", K(tenant));
+        LOGMINER_STDOUT("oblogminer only support user tenant(e.g., user@tenant).\n");
+      }
+      // check password not empty
+      if (OB_SUCC(ret)) {
+        if (0 == strcmp(password_, "")) {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_ERROR("password should not be empty string", KCSTRING(password_));
+          LOGMINER_STDOUT("password should not be empty string\n");
+        }
+      }
+    }
+    if (OB_SUCC(ret) && nullptr != tenant) {
+      bool validate = true;
+      if (OB_FAIL(validate_table_list_(tenant, validate))) {
+        LOG_ERROR("validate table list failed", K(ret), KCSTRING(tenant), KCSTRING(table_list_));
+        LOGMINER_STDOUT("parse table_list failed\n");
+      } else {
+        if (!validate) {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_ERROR("tenant in table_list must match the tenant specified in user-name", KCSTRING(tenant));
+          LOGMINER_STDOUT("tenant in table_list must match the tenant specified in user-name\n");
+        }
+      }
+    }
+    if (OB_NOT_NULL(user_name_buf)) {
+      ob_free(user_name_buf);
+    }
+  }
+
+  return ret;
+}
+
+// check tenant name in table_list
+int AnalyzerArgs::validate_table_list_(const char *tenant_name, bool &validate) const
+{
+  int ret = OB_SUCCESS;
+  const char pattern_delimiter = '|';
+  const char name_delimiter = '.';
+  bool done = false;
+  int buffer_size = strlen(table_list_) + 1;
+  char *buffer = nullptr;
+  ObString remain;
+  ObString cur_pattern;
+  ObString tenant_pattern;
+  ObString database_pattern;
+  ObString table_pattern;
+  if (OB_ISNULL(buffer = reinterpret_cast<char*>(ob_malloc(buffer_size, "LogMnrTableList")))) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("err alloc string buffer", K(ret), K(buffer_size));
+  } else {
+    MEMCPY(buffer, table_list_, buffer_size-1);
+    buffer[buffer_size-1] = '\0';
+    remain.assign_ptr(buffer, buffer_size);
+  }
+  while (OB_SUCCESS == ret && !done) {
+    // Split Pattern & get current pattern.
+    cur_pattern = remain.split_on(pattern_delimiter);
+    if (cur_pattern.empty()) {
+      cur_pattern = remain;
+      done = true;
+    }
+    if (OB_SUCC(ret)) {
+      tenant_pattern = cur_pattern.split_on(name_delimiter);
+      if (tenant_pattern.empty()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("invalid argment", K(ret), K(cur_pattern));
+        break;
+      }
+      if (OB_SUCC(ret)) {
+        if (0 != tenant_pattern.compare(tenant_name) && 0 != tenant_pattern.compare("*")) {
+          validate = false;
+          LOG_INFO("tenant in table_list must match the tenant specified in user-name",
+              KCSTRING(tenant_name), K(tenant_pattern));
+          break;
+        }
+      }
+    }
+    if (OB_SUCC(ret)) {
+      database_pattern = cur_pattern.split_on(name_delimiter);
+      if (database_pattern.empty()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("invalid argument", KR(ret), K(cur_pattern));
+        break;
+      }
+    }
+    // Table name.
+    if (OB_SUCC(ret)) {
+      table_pattern = cur_pattern;
+      if (table_pattern.empty()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("invalid argment", KR(ret), K(cur_pattern));
+        break;
+      }
+    }
+  } // while
+  if (OB_NOT_NULL(buffer)) {
+    ob_free(buffer);
+  }
+  return ret;
+}
+
+int AnalyzerArgs::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
+{
+  int ret = OB_SUCCESS;
+  // make sure there is no nullptr among class members
+  if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", CLUSTER_ADDR_KEY,
+      empty_str_wrapper(cluster_addr_)))) {
+    LOG_ERROR("failed to print cluster_addr", KCSTRING(cluster_addr_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", USER_NAME_KEY,
+      empty_str_wrapper(user_name_)))) {
+    LOG_ERROR("failed to print user_name", KCSTRING(user_name_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", PASSWORD_KEY,
+      empty_str_wrapper(password_)))) {
+    LOG_ERROR("failed to print password", K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", ARCHIVE_DEST_KEY,
+      empty_str_wrapper(archive_dest_)))) {
+    LOG_ERROR("failed to print archive_dest", K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", TABLE_LIST_KEY,
+      empty_str_wrapper(table_list_)))) {
+    LOG_ERROR("failed to print table_list", KCSTRING(table_list_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", COLUMN_COND_KEY,
+      empty_str_wrapper(column_cond_)))) {
+    LOG_ERROR("failed to print column cond", KCSTRING(column_cond_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", OPERATIONS_KEY,
+      empty_str_wrapper(operations_)))) {
+    LOG_ERROR("failed to print operations", KCSTRING(operations_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", OUTPUT_DST_KEY,
+      empty_str_wrapper(output_dst_)))) {
+    LOG_ERROR("failed to print output_dst", K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", LOG_LEVEL_KEY,
+      empty_str_wrapper(log_level_)))) {
+    LOG_ERROR("failed to print log_level", KCSTRING(log_level_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n", START_TIME_US_KEY,
+      start_time_us_))) {
+    LOG_ERROR("failed to print start_time_us", K(start_time_us_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n", END_TIME_US_KEY,
+      end_time_us_))) {
+    LOG_ERROR("failed to print end_time_us", K(end_time_us_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", TIMEZONE_KEY,
+      timezone_))) {
+    LOG_ERROR("failed to print timezone", KCSTRING(timezone_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", RECORD_FORMAT_KEY,
+      record_file_format_str(record_format_)))) {
+    LOG_ERROR("failed to print record format", KCSTRING(record_file_format_str(record_format_)),
+        K(buf_len), K(pos));
+  }
+  return ret;
+}
+
+int AnalyzerArgs::deserialize(const char *buf, const int64_t data_len, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+  ObString record_format_str;
+  if (OB_FAIL(parse_line(CLUSTER_ADDR_KEY, buf, data_len, pos, alloc_, cluster_addr_))) {
+    LOG_ERROR("failed to get cluster_addr", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(USER_NAME_KEY, buf, data_len, pos, alloc_, user_name_))) {
+    LOG_ERROR("failed to get user_name", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(PASSWORD_KEY, buf, data_len, pos, alloc_, password_))) {
+    LOG_ERROR("failed to get password", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(ARCHIVE_DEST_KEY, buf, data_len, pos, alloc_, archive_dest_))) {
+    LOG_ERROR("failed to get archive_dest", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(TABLE_LIST_KEY, buf, data_len, pos, alloc_, table_list_))) {
+    LOG_ERROR("failed to get table_list", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(COLUMN_COND_KEY, buf, data_len, pos, alloc_, column_cond_))) {
+    LOG_ERROR("failed to get column_cond", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(OPERATIONS_KEY, buf, data_len, pos, alloc_, operations_))) {
+    LOG_ERROR("failed to get operations", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(OUTPUT_DST_KEY, buf, data_len, pos, alloc_, output_dst_))) {
+    LOG_ERROR("failed to get output_dst", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(LOG_LEVEL_KEY, buf, data_len, pos, alloc_, log_level_))) {
+    LOG_ERROR("failed to get log_level", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(START_TIME_US_KEY, buf, data_len, pos, start_time_us_))) {
+    LOG_ERROR("failed to get start_time_us", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(END_TIME_US_KEY, buf, data_len, pos, end_time_us_))) {
+    LOG_ERROR("failed to get end_time_us", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(TIMEZONE_KEY, buf, data_len, pos, alloc_, timezone_))) {
+    LOG_ERROR("failed to get timezone", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(RECORD_FORMAT_KEY, buf, data_len, pos, record_format_str))) {
+    LOG_ERROR("failed to ger record_format", K(data_len), K(pos));
+  }
+  if (OB_SUCC(ret)) {
+    record_format_ = get_record_file_format(record_format_str);
+  }
+  return ret;
+}
+
+int64_t AnalyzerArgs::get_serialize_size() const
+{
+  int64_t size = 0;
+  char buf[30] = {0};
+  int64_t start_time_us_len = snprintf(buf, sizeof(buf), "%ld", start_time_us_);
+  int64_t end_time_us_len = snprintf(buf, sizeof(buf), "%ld", end_time_us_);
+  // need add "=" and "\n"
+  size += strlen(CLUSTER_ADDR_KEY) + 1 + strlen(empty_str_wrapper(cluster_addr_)) + 1;
+  size += strlen(USER_NAME_KEY) + 1 + strlen(empty_str_wrapper(user_name_)) + 1;
+  size += strlen(PASSWORD_KEY) + 1 + strlen(empty_str_wrapper(password_)) + 1;
+  size += strlen(ARCHIVE_DEST_KEY) + 1 + strlen(empty_str_wrapper(archive_dest_)) + 1;
+  size += strlen(TABLE_LIST_KEY) + 1 + strlen(empty_str_wrapper(table_list_)) + 1;
+  size += strlen(COLUMN_COND_KEY) + 1 + strlen(empty_str_wrapper(column_cond_)) + 1;
+  size += strlen(OPERATIONS_KEY) + 1 + strlen(empty_str_wrapper(operations_)) + 1;
+  size += strlen(OUTPUT_DST_KEY) + 1 + strlen(empty_str_wrapper(output_dst_)) + 1;
+  size += strlen(LOG_LEVEL_KEY) + 1 + strlen(empty_str_wrapper(log_level_)) + 1;
+  size += strlen(START_TIME_US_KEY) + 1 + start_time_us_len + 1;
+  size += strlen(END_TIME_US_KEY) + 1 + end_time_us_len + 1;
+  size += strlen(TIMEZONE_KEY) + 1 + strlen(empty_str_wrapper(timezone_)) + 1;
+  size += strlen(RECORD_FORMAT_KEY) + 1 + strlen(record_file_format_str(record_format_)) + 1;
+  return size;
+}
+
+/////////////////////////////////////// FlashbackerArgs ///////////////////////////////////////
+
+void FlashbackerArgs::reset()
+{
+  cluster_addr_ = nullptr;
+  user_name_ = nullptr;
+  password_ = nullptr;
+  recovery_path_ = nullptr;
+  tgt_table_ = nullptr;
+  src_table_ = nullptr;
+  progress_dst_ = nullptr;
+  log_level_ = nullptr;
+  start_time_us_ = OB_INVALID_TIMESTAMP;
+  end_time_us_ = OB_INVALID_TIMESTAMP;
+}
+
+int FlashbackerArgs::init(const ObLogMinerCmdArgs &args)
+{
+  int ret = OB_SUCCESS;
+  if (!is_flashback_mode(args.mode_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("incorrect cmd args to init flashbacker args", K(args));
+  } else {
+    cluster_addr_ = args.cluster_addr_;
+    user_name_ = args.user_name_;
+    password_ = args.password_;
+    recovery_path_ = args.recovery_path_;
+    tgt_table_ = args.target_table_;
+    src_table_ = args.source_table_;
+    progress_dst_ = args.output_;
+    log_level_ = args.log_level_;
+
+    if (OB_FAIL(logminer_str2ll(args.start_time_, start_time_us_))) {
+      LOG_ERROR("parse start time for FlashbackerArgs failed", K(args), K(start_time_us_));
+    } else if (OB_FAIL(logminer_str2ll(args.end_time_, end_time_us_))) {
+      LOG_ERROR("parse end time for FlashbackerArgs failed", K(args), K(end_time_us_));
+    } else if (OB_FAIL(validate())) {
+      LOG_ERROR("validate FlashbackerArgs failed after initialize all arguments", K(args), KPC(this));
+    } else {
+      // succ
+      LOG_INFO("finished to init flashbacker args", KPC(this));
+    }
+  }
+
+  return ret;
+}
+
+int FlashbackerArgs::validate() const
+{
+  int ret = OB_SUCCESS;
+
+  if (nullptr == cluster_addr_ || nullptr == user_name_ || nullptr == password_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("cluster_addr, user_name or password is empty", KPC(this));
+  } else if (nullptr == tgt_table_ || nullptr == src_table_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("target-table or source-table is empty", KPC(this));
+  } else if (nullptr == progress_dst_ || nullptr == recovery_path_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("recovery path or progress_dst is empty", KPC(this));
+  } else if (OB_INVALID_TIMESTAMP == start_time_us_ ||
+             OB_INVALID_TIMESTAMP == end_time_us_ ||
+             start_time_us_ >= end_time_us_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("start_time or end_time is not valid", K(start_time_us_), K(end_time_us_), KPC(this));
+  } else {
+    // succ
+  }
+
+  return ret;
+}
+
+int FlashbackerArgs::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
+{
+  int ret = OB_SUCCESS;
+  // TODO
+  return ret;
+}
+
+int FlashbackerArgs::deserialize(const char *buf, const int64_t data_len, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+  // TODO
+  return ret;
+}
+
+int64_t FlashbackerArgs::get_serialize_size() const
+{
+  int64_t size = 0;
+  // TODO
+  return size;
+}
+
+/////////////////////////////////////// ObLogMinerArgs ///////////////////////////////////////
+const char *ObLogMinerArgs::DEFAULT_LOG_LEVEL = "ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN";
+const char *ObLogMinerArgs::LOGMINER_LOG_FILE = "oblogminer.log";
+const char *ObLogMinerArgs::DEFAULT_LOGMNR_TIMEZONE = "+8:00";
+const char *ObLogMinerArgs::DEFAULT_LOGMNR_OPERATIONS = "insert|delete|update";
+const char *ObLogMinerArgs::DEFAULT_LOGMNR_TABLE_LIST = "*.*.*";
+const char *ObLogMinerArgs::DEFAULT_LOGMNR_FORMAT = "csv";
+
+
+void ObLogMinerArgs::reset()
+{
+  mode_ = LogMinerMode::UNKNOWN;
+  verbose_ = false;
+  print_usage_ = false;
+  timezone_ = nullptr;
+  analyzer_args_.reset();
+  flashbacker_args_.reset();
+}
+
+int ObLogMinerArgs::validate() const
+{
+  int ret = OB_SUCCESS;
+  if (! is_logminer_mode_valid(mode_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("logminer mode is invalid", K(mode_));
+  } else if (is_analysis_mode(mode_)) {
+    ret = analyzer_args_.validate();
+  } else if (is_flashback_mode(mode_)) {
+    ret = flashbacker_args_.validate();
+  }
+
+  if (OB_FAIL(ret)) {
+    LOG_ERROR("ObLogMiner found some arguments invalid", KPC(this));
+  }
+
+  return ret;
+}
+
+int ObLogMinerArgs::init(int argc, char *argv[])
+{
+  int ret = OB_SUCCESS;
+  ObLogMinerCmdArgs args;
+  if (OB_FAIL(args.init(argc, argv))) {
+    LOG_ERROR("init ObLogMinerCmdArgs failed", K(argc));
+  } else if (args.print_usage_) {
+    print_usage_ = args.print_usage_;
+  } else if (! is_logminer_mode_valid(args.mode_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("ObLogMinerArgs got invalid mode argument", K(args.mode_),
+        "mode", logminer_mode_to_str(args.mode_));
+  } else {
+    mode_ = args.mode_;
+    verbose_ = args.verbose_;
+    timezone_ = args.timezone_ == nullptr ? DEFAULT_LOGMNR_TIMEZONE : args.timezone_;
+
+    if (nullptr == args.log_level_) {
+      args.log_level_ = DEFAULT_LOG_LEVEL;
+    }
+
+    if (OB_FAIL(ret)) {
+      LOG_ERROR("failed to set mod log level previously, unexpected");
+    } else if (is_analysis_mode(args.mode_)) {
+      if (OB_FAIL(analyzer_args_.init(args))) {
+        LOG_ERROR("failed to ini analyzer args", K(args.mode_), K(analyzer_args_), K(args));
+      }
+    } else if (is_flashback_mode(args.mode_)) {
+      if (OB_FAIL(flashbacker_args_.init(args))) {
+        LOG_ERROR("failed to ini flashbacker args", K(args.mode_), K(flashbacker_args_), K(args));
+      }
+    } else {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("vaild logminer mode is ignored, which is unexpected", K(args.mode_),
+          "mode", logminer_mode_to_str(args.mode_));
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerArgs::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf + pos, buf_len, pos, "mode=%s\n", logminer_mode_to_str(mode_)))) {
+    LOG_ERROR("failed to serialize mode", K(mode_));
+  } else if (is_analysis_mode(mode_)) {
+    if (OB_FAIL(analyzer_args_.serialize(buf, buf_len, pos))) {
+      LOG_ERROR("analyzer_args failed to serialize", K(buf), K(buf_len), K(pos), K(analyzer_args_));
+    }
+  } else if (is_flashback_mode(mode_)) {
+    if (OB_FAIL(flashbacker_args_.serialize(buf, buf_len, pos))) {
+      LOG_ERROR("flashbacker_args failed to serialize", K(buf), K(buf_len), K(pos), K(flashbacker_args_));
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerArgs::deserialize(const char *buf, const int64_t data_len, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+  ObString mode_str;
+  if (OB_FAIL(parse_line("mode", buf, data_len, pos, mode_str))) {
+    LOG_ERROR("failed to get mode string", K(data_len), K(pos));
+  } else {
+    mode_ = get_logminer_mode(mode_str);
+    if (is_analysis_mode(mode_)) {
+      if (OB_FAIL(analyzer_args_.deserialize(buf, data_len, pos))) {
+        LOG_ERROR("failed to deserialize analyzer arguments", K(data_len), K(pos));
+      }
+    } else {
+      ret = OB_INVALID_DATA;
+      LOG_ERROR("get invalid mode when deserializing logminer args", K(mode_str), K(mode_));
+    }
+  }
+  return ret;
+}
+
+int64_t ObLogMinerArgs::get_serialize_size() const
+{
+  int64_t size = 0;
+
+  size += strlen("mode=") + strlen(logminer_mode_to_str(mode_)) + strlen("\n");
+
+  if (is_analysis_mode(mode_)) {
+    size += analyzer_args_.get_serialize_size();
+  } else if (is_flashback_mode(mode_)) {
+    size += flashbacker_args_.get_serialize_size();
+  }
+
+  return size;
+}
+
+
+int64_t ObLogMinerArgs::to_string(char *buf, const int64_t buf_len) const
+{
+  int64_t pos = 0;
+  databuff_print_kv(buf, buf_len, pos, "mode", mode_);
+  if (is_analysis_mode(mode_)) {
+    databuff_print_kv(buf, buf_len, pos, "analyzer_args", analyzer_args_);
+  } else if (is_flashback_mode(mode_)) {
+    databuff_print_kv(buf, buf_len, pos, "flashbacker_args", flashbacker_args_);
+  } else {
+    // invalid mode
+  }
+  return pos;
+}
+
+
+
+}
+}

src/logservice/logminer/ob_log_miner_args.h

@@ -0,0 +1,201 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCENABASE_LOG_MINER_ARGS_H_
+#define OCENABASE_LOG_MINER_ARGS_H_
+
+#include <cstdint>
+#include "lib/allocator/page_arena.h"     // ObArenaAllocator
+#include "lib/string/ob_string_buffer.h"
+#include "lib/utility/ob_print_utils.h"
+#include "ob_log_miner_mode.h"
+#include "ob_log_miner_record_file_format.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerCmdArgs {
+public:
+  static void print_usage(const char *prog_name);
+public:
+  ObLogMinerCmdArgs()
+  { reset(); }
+
+  int init(int argc, char *argv[]);
+
+  void reset();
+
+  TO_STRING_KV(
+    K(mode_),
+    KCSTRING(cluster_addr_),
+    KCSTRING(user_name_),
+    KCSTRING(table_list_),
+    KCSTRING(column_cond_),
+    KCSTRING(operations_),
+    KCSTRING(target_table_),
+    KCSTRING(source_table_),
+    KCSTRING(start_time_),
+    KCSTRING(end_time_),
+    KCSTRING(log_level_),
+    KCSTRING(timezone_),
+    KCSTRING(record_format_),
+    K(verbose_),
+    K(print_usage_)
+  );
+
+public:
+  LogMinerMode  mode_;
+  const char    *cluster_addr_;
+  const char    *user_name_;
+  const char    *password_;
+  const char    *archive_dest_;
+  const char    *table_list_;
+  const char    *column_cond_;
+  const char    *operations_;
+  const char    *output_;
+  const char    *recovery_path_;
+  const char    *target_table_;
+  const char    *source_table_;
+  const char    *start_time_;
+  const char    *end_time_;
+  const char    *log_level_;
+  const char    *timezone_;
+  const char    *record_format_;
+  bool          verbose_;
+  bool          print_usage_;
+private:
+  int validate_() const;
+};
+
+class AnalyzerArgs {
+  static const char *CLUSTER_ADDR_KEY;
+  static const char *USER_NAME_KEY;
+  static const char *PASSWORD_KEY;
+  static const char *ARCHIVE_DEST_KEY;
+  static const char *TABLE_LIST_KEY;
+  static const char *COLUMN_COND_KEY;
+  static const char *OPERATIONS_KEY;
+  static const char *OUTPUT_DST_KEY;
+  static const char *LOG_LEVEL_KEY;
+  static const char *START_TIME_US_KEY;
+  static const char *END_TIME_US_KEY;
+  static const char *TIMEZONE_KEY;
+  static const char *RECORD_FORMAT_KEY;
+public:
+  AnalyzerArgs() { reset(); }
+  int init(const ObLogMinerCmdArgs &args);
+  int validate() const;
+  void reset();
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  TO_STRING_KV(
+    KCSTRING(cluster_addr_),
+    KCSTRING(user_name_),
+    KCSTRING(table_list_),
+    KCSTRING(column_cond_),
+    KCSTRING(operations_),
+    KCSTRING(log_level_),
+    K(start_time_us_),
+    K(end_time_us_),
+    KCSTRING(timezone_),
+    K(record_format_)
+  );
+
+public:
+  char *cluster_addr_;
+  char *user_name_;
+  char *password_;
+  char *archive_dest_;
+  char *table_list_;
+  char *column_cond_;
+  char *operations_;
+  char *output_dst_;
+  char *log_level_;
+  int64_t start_time_us_;
+  int64_t end_time_us_;
+  char *timezone_;
+  RecordFileFormat record_format_;
+private:
+  int validate_table_list_(const char *tenant_name, bool &validate) const;
+private:
+  ObArenaAllocator alloc_;
+};
+
+class FlashbackerArgs {
+public:
+  FlashbackerArgs() { reset(); }
+  int init(const ObLogMinerCmdArgs &args);
+  int validate() const;
+  void reset();
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  TO_STRING_KV(
+    KCSTRING(cluster_addr_),
+    KCSTRING(user_name_),
+    KCSTRING(tgt_table_),
+    KCSTRING(src_table_),
+    KCSTRING(progress_dst_),
+    KCSTRING(log_level_),
+    K(start_time_us_),
+    K(end_time_us_)
+  );
+public:
+  const char *cluster_addr_;
+  const char *user_name_;
+  const char *password_;
+  const char *recovery_path_;
+  const char *tgt_table_;
+  const char *src_table_;
+  const char *progress_dst_;
+  const char *log_level_;
+  int64_t start_time_us_;
+  int64_t end_time_us_;
+};
+
+class ObLogMinerArgs {
+public:
+  static const char *DEFAULT_LOG_LEVEL;
+  static const char *LOGMINER_LOG_FILE;
+  static const char *DEFAULT_LOGMNR_TIMEZONE;
+  static const char *DEFAULT_LOGMNR_OPERATIONS;
+  static const char *DEFAULT_LOGMNR_TABLE_LIST;
+  static const char *DEFAULT_LOGMNR_FORMAT;
+public:
+  ObLogMinerArgs() { reset(); }
+  ~ObLogMinerArgs() { reset(); }
+  int init(int argc, char *argv[]);
+  int validate() const;
+  void reset();
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  DECLARE_TO_STRING;
+
+public:
+  LogMinerMode mode_;
+  bool verbose_;
+  bool print_usage_;
+  const char *timezone_;
+  AnalyzerArgs analyzer_args_;
+  FlashbackerArgs flashbacker_args_;
+private:
+  DISALLOW_COPY_AND_ASSIGN(ObLogMinerArgs);
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_batch_record.cpp

@@ -0,0 +1,219 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_record_converter.h"
+#include "ob_log_miner_batch_record.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+void ObLogMinerBatchRecord::reset()
+{
+  range_.reset();
+  begin_record_.reset();
+  buf_.reset();
+  trans_data_record_count_ = 0;
+  total_data_record_count_ = 0;
+  written_record_count_ = 0;
+  first_access_ts_ = OB_INVALID_TIMESTAMP;
+  last_trans_end_ts_ = OB_INVALID_TIMESTAMP;
+
+  seq_no_ = -1;
+  file_id_ = -1;
+  file_offset_ = -1;
+  is_file_end_ = false;
+
+  freezed_ = false;
+}
+
+int ObLogMinerBatchRecord::init_last_trans_end_ts(const int64_t commit_ts)
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_INVALID_TIMESTAMP == last_trans_end_ts_) {
+    last_trans_end_ts_ = commit_ts;
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("last trans end ts rollbacked, unexpected", K(last_trans_end_ts_), K(commit_ts));
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecord::append_record(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  if (freezed_) {
+    ret = OB_BUF_NOT_ENOUGH;
+    LOG_WARN("batch record has been freezed, need retry", K(freezed_), "buf_len", buf_.length(),
+        "buf_cap", buf_.capacity());
+  } else {
+    const RecordType record_type = record.get_record_type();
+    if (OB_INVALID_TIMESTAMP == first_access_ts_) {
+      first_access_ts_ = ObTimeUtility::current_time();
+    }
+    switch (record_type) {
+      case EBEGIN: {
+        ret = handle_begin_record_(record);
+        break;
+      }
+
+      case ECOMMIT: {
+        ret = handle_commit_record_(record);
+        break;
+      }
+
+      case EDDL: {
+        ret = handle_ddl_record_(record);
+        break;
+      }
+
+      case EINSERT:
+      case EUPDATE:
+      case EDELETE: {
+        ret = handle_dml_record_(record);
+        break;
+      }
+
+      case HEARTBEAT: {
+        ret = handle_heartbeat_record_(record);
+        break;
+      }
+
+      default: {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("get unexpected record type", K(record));
+      }
+    }
+
+    if (OB_FAIL(ret)) {
+      LOG_ERROR("append record to buffer failed", K(record));
+    } else {
+      const int64_t record_commit_ts = record.get_commit_scn().convert_to_ts();
+      update_progress_range_(record_commit_ts);
+      total_data_record_count_++;
+      if (is_trans_end_record_type(record_type)) {
+        last_trans_end_ts_ = record_commit_ts;
+      }
+      if (buf_.length() >= BATCH_RECORD_THRESHOLD) {
+        freezed_ = true;
+      }
+    }
+  }
+
+
+  return ret;
+}
+
+int ObLogMinerBatchRecord::handle_begin_record_(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  if (begin_record_.is_inited()) {
+    ret = OB_ERR_DEFENSIVE_CHECK;
+    LOG_ERROR("get duplicate begin record, unexpected", K(begin_record_), K(record), KPC(this));
+  } else {
+    begin_record_.copy_base_info(record);
+  }
+  return ret;
+}
+
+int ObLogMinerBatchRecord::handle_commit_record_(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  if (begin_record_.is_inited() && 0 == trans_data_record_count_) {
+    // exit, no need to write begin & commit record
+    LOG_TRACE("all data in trans is filtered", K(record), K(begin_record_), KPC(this));
+    begin_record_.reset();
+  } else if (OB_FAIL(write_record_(record))) {
+    LOG_ERROR("failed to write commit record into buffer", K(record), KPC(this));
+  } else {
+    trans_data_record_count_ = 0;
+
+    if (begin_record_.is_inited()) {
+      begin_record_.reset();
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecord::handle_dml_record_(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  if (begin_record_.is_inited()) {
+    if (OB_FAIL(write_record_(begin_record_))) {
+      LOG_ERROR("write begin record into buffer failed", K(begin_record_), KPC(this));
+    } else {
+      begin_record_.reset();
+    }
+  }
+
+  if (OB_SUCC(ret) && OB_FAIL(write_record_(record))) {
+    LOG_ERROR("failed to write dml record", K(record), KPC(this));
+  } else {
+    trans_data_record_count_++;
+  }
+  return ret;
+}
+
+int ObLogMinerBatchRecord::handle_ddl_record_(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(write_record_(record))) {
+    LOG_ERROR("write ddl record failed", K(record), KPC(this));
+  }
+  return ret;
+}
+
+int ObLogMinerBatchRecord::handle_heartbeat_record_(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+
+  return ret;
+}
+
+void ObLogMinerBatchRecord::update_progress_range_(const int64_t progress)
+{
+  if (OB_INVALID_TIMESTAMP == range_.min_commit_ts_) {
+    range_.min_commit_ts_ = progress;
+  }
+
+  if (OB_INVALID_TIMESTAMP == range_.max_commit_ts_ || progress >= range_.max_commit_ts_){
+    range_.max_commit_ts_ = progress;
+  }
+}
+
+int ObLogMinerBatchRecord::write_record_(const ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  bool is_written = false;
+  if (OB_ISNULL(converter_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get an invalid converter when write_record", K(record));
+  } else if (OB_FAIL(converter_->write_record(record, buf_, is_written))) {
+    LOG_ERROR("converter failed to convert record", K(record), "buf_len", buf_.length(),
+        "buf_cap", buf_.capacity());
+  } else {
+    if (is_written) {
+      written_record_count_++;
+    }
+  }
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_batch_record.h

@@ -0,0 +1,200 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_BATCH_RECORD_
+#define OCEANBASE_LOG_MINER_BATCH_RECORD_
+
+#include "lib/allocator/ob_allocator.h"
+#include "lib/string/ob_string_buffer.h"
+#include "ob_log_miner_record.h"
+#include "ob_log_miner_file_manager.h"
+#include <cstdint>
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerRecordConverter;
+
+class ObLogMinerBatchRecord: public ObLogMinerRecyclableTask
+{
+public:
+  static const int64_t BATCH_RECORD_THRESHOLD = 2L * 1024 * 1024; // 2MB
+private:
+  ObLogMinerBatchRecord():
+      ObLogMinerRecyclableTask(TaskType::BATCH_RECORD),
+      alloc_(nullptr),
+      converter_(nullptr),
+      range_(),
+      begin_record_(),
+      buf_(),
+      trans_data_record_count_(0),
+      total_data_record_count_(0),
+      written_record_count_(0),
+      first_access_ts_(OB_INVALID_TIMESTAMP),
+      last_trans_end_ts_(OB_INVALID_TIMESTAMP),
+      seq_no_(-1),
+      file_id_(-1),
+      file_offset_(-1),
+      is_file_end_(false),
+      freezed_(false)
+      {}
+public:
+  ObLogMinerBatchRecord(ObIAllocator *alloc, ILogMinerRecordConverter *converter):
+      ObLogMinerBatchRecord()
+  {
+    set_allocator(alloc);
+    set_record_converter(converter);
+  }
+  ~ObLogMinerBatchRecord()
+  {
+    reset();
+    converter_ = nullptr;
+    alloc_ = nullptr;
+  }
+
+  void reset();
+
+  void set_allocator(ObIAllocator *alloc)
+  {
+    alloc_ = alloc;
+    buf_.set_allocator(alloc_);
+  }
+
+  int init_last_trans_end_ts(const int64_t commit_ts);
+
+  int64_t get_last_trans_end_ts() const
+  {
+    return last_trans_end_ts_;
+  }
+
+  void set_record_converter(ILogMinerRecordConverter *converter)
+  {
+    converter_ = converter;
+  }
+
+  int append_record(const ObLogMinerRecord &record);
+  void get_data(const char *&data, int64_t &data_len) const {
+    data = buf_.ptr();
+    data_len = buf_.length();
+  }
+
+  int64_t get_total_record_count() const {
+    return total_data_record_count_;
+  }
+
+  int64_t get_cur_trans_record_count() const {
+    return trans_data_record_count_;
+  }
+
+   int64_t get_written_record_count() const {
+    return written_record_count_;
+  }
+
+  const ObLogMinerProgressRange &get_progress_range() const {
+    return range_;
+  }
+  int64_t get_first_access_ts() const {
+    return first_access_ts_;
+  }
+  bool is_freezed() const {
+    return freezed_;
+  }
+
+  void set_seq_no(const int64_t seq_no) {
+    seq_no_ = seq_no;
+  }
+
+  int64_t get_seq_no() const {
+    return seq_no_;
+  }
+
+  void set_file_id(const int64_t file_id) {
+    file_id_ = file_id;
+  }
+
+  int64_t get_file_id() const {
+    return file_id_;
+  }
+
+  void set_file_offset(const int64_t file_offset) {
+    file_offset_ = file_offset;
+  }
+
+  int64_t get_file_offset() const {
+    return file_offset_;
+  }
+
+  void set_file_end() {
+    is_file_end_ = true;
+  }
+
+  bool is_file_end() const {
+    return is_file_end_;
+  }
+
+  TO_STRING_KV(
+    K(range_),
+    K(begin_record_),
+    K(trans_data_record_count_),
+    K(total_data_record_count_),
+    K(written_record_count_),
+    K(first_access_ts_),
+    K(seq_no_),
+    K(file_id_),
+    K(file_offset_),
+    K(is_file_end_),
+    K(freezed_)
+  );
+
+private:
+  int handle_begin_record_(const ObLogMinerRecord &record);
+
+  int handle_dml_record_(const ObLogMinerRecord &record);
+
+  int handle_ddl_record_(const ObLogMinerRecord &record);
+
+  int handle_commit_record_(const ObLogMinerRecord &record);
+
+  int handle_heartbeat_record_(const ObLogMinerRecord &record);
+
+  int write_record_(const ObLogMinerRecord &record);
+
+  void update_progress_range_(const int64_t progress);
+
+private:
+  common::ObIAllocator *alloc_;
+  ILogMinerRecordConverter *converter_;
+
+  ObLogMinerProgressRange range_;
+  ObLogMinerRecord begin_record_;
+  ObStringBuffer buf_;
+  int64_t trans_data_record_count_;
+  int64_t total_data_record_count_;
+  int64_t written_record_count_;
+  int64_t first_access_ts_; // in milliseconds
+
+  int64_t last_trans_end_ts_; // in milliseconds
+
+  int64_t seq_no_;
+  int64_t file_id_;
+  int64_t file_offset_;
+  bool is_file_end_;
+
+  bool freezed_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_batch_record_writer.cpp

@@ -0,0 +1,383 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "lib/container/ob_ext_ring_buffer.h"
+#include "ob_log_miner_batch_record_writer.h"
+#include "ob_log_miner_batch_record.h"
+#include "ob_log_miner_file_manager.h"
+#include "ob_log_miner_resource_collector.h"
+#include "ob_log_miner_data_manager.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+ObLogMinerBatchRecordWriter::ObLogMinerBatchRecordWriter():
+    is_inited_(false),
+    current_file_id_(-1),
+    current_file_offset_(-1),
+    batch_record_sw_(),
+    record_count_(0),
+    batch_record_count_(0),
+    data_manager_(nullptr),
+    resource_collector_(nullptr),
+    file_manager_(nullptr),
+    err_handle_(nullptr) { }
+
+int ObLogMinerBatchRecordWriter::init(ILogMinerDataManager *data_mgr,
+    ILogMinerResourceCollector *res_collector,
+    ILogMinerFileManager *file_mgr,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+  const int64_t SLIDING_WINDOW_SIZE = 2L << 20;
+
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("BatchRecordWriter has been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data_mgr) || OB_ISNULL(res_collector) || OB_ISNULL(file_mgr)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid argument when initializing BatchRecordWriter", K(data_mgr),
+        K(res_collector), K(file_mgr));
+  } else if (OB_FAIL(BatchWriterThreadPool::init(BATCH_WRITER_THREAD_NUM, BATCH_WRITER_QUEUE_SIZE))) {
+    LOG_ERROR("BatchWriterThreadPool failed to init");
+  } else if (OB_FAIL(batch_record_sw_.init(1, SLIDING_WINDOW_SIZE))) {
+    LOG_ERROR("batch record sliding window failed to initialize", K(SLIDING_WINDOW_SIZE));
+  } else {
+    current_file_id_ = 0;
+    current_file_offset_ = 0;
+    data_manager_ = data_mgr;
+    resource_collector_ = res_collector;
+    file_manager_ = file_mgr;
+    err_handle_ = err_handle;
+    is_inited_ = true;
+    LOG_INFO("ObLogMinerBatchRecordWriter finished to init", K(data_manager_), K(resource_collector_),
+        K(file_manager_));
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecordWriter::start()
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_FAIL(BatchWriterThreadPool::start())) {
+    LOG_ERROR("BatchWriterThreadPool failed to start");
+  } else if (OB_FAIL(lib::ThreadPool::start())) {
+    LOG_ERROR("thread pool in batchrecord writer failed to start");
+  } else {
+    LOG_INFO("ObLogMinerBatchRecordWriter starts");
+  }
+
+  return ret;
+}
+
+void ObLogMinerBatchRecordWriter::stop()
+{
+  BatchWriterThreadPool::mark_stop_flag();
+  lib::ThreadPool::stop();
+  LOG_INFO("ObLogMinerBatchRecordWriter stopped");
+}
+
+void ObLogMinerBatchRecordWriter::wait()
+{
+  BatchWriterThreadPool::stop();
+  lib::ThreadPool::wait();
+  LOG_INFO("ObLogMinerBatchRecordWriter finished to wait");
+}
+
+void ObLogMinerBatchRecordWriter::destroy()
+{
+  if (IS_INIT) {
+    BatchWriterThreadPool::destroy();
+    lib::ThreadPool::destroy();
+    current_file_id_ = -1;
+    current_file_offset_ = -1;
+    data_manager_ = nullptr;
+    resource_collector_ = nullptr;
+    file_manager_ = nullptr;
+    err_handle_ = nullptr;
+    record_count_ = 0;
+    batch_record_count_ = 0;
+    batch_record_sw_.destroy();
+    is_inited_ = false;
+  }
+}
+
+
+// only one thread will call this method
+int ObLogMinerBatchRecordWriter::push(ObLogMinerBatchRecord *record)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_));
+  } else if (OB_ISNULL(record)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid record when trying to push record into BatchWriter", K(record));
+  } else {
+    LOG_TRACE("ObLogMinerBatchRecordWriter push batch record into queue", KPC(record));
+    const int64_t DATA_OP_TIMEOUT = 1L * 1000 * 1000;
+    static volatile int64_t seq_no = 0;
+    const int64_t assigned_seq_no = ATOMIC_AAF(&seq_no, 1);
+    if (OB_FAIL(set_file_info_(*record, assigned_seq_no))) {
+      LOG_ERROR("set file info for record failed", K(record), K(current_file_id_), K(current_file_offset_));
+    } else {
+      ATOMIC_AAF(&record_count_, record->get_total_record_count());
+      ATOMIC_INC(&batch_record_count_);
+      RETRY_FUNC(is_stoped(), *(static_cast<ObMQThread*>(this)), push, record, record->get_file_id(), DATA_OP_TIMEOUT);
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecordWriter::handle(void *data, const int64_t thread_index, volatile bool &stop_flag)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid data when handling data in BatchWriter", K(data));
+  } else {
+    ObLogMinerBatchRecord *record = static_cast<ObLogMinerBatchRecord*>(data);
+    const char *data = nullptr;
+    int64_t data_len = 0;
+    record->get_data(data, data_len);
+    if (OB_FAIL(file_manager_->append_records(*record))) {
+      LOG_ERROR("write file through file manager failed", KPC(record), K(data_len));
+    } else if (OB_FAIL(generate_meta_flush_task_(*record))) {
+      LOG_ERROR("generate meta flush task failed", K(record));
+    } else {
+      ATOMIC_SAF(&record_count_, record->get_total_record_count());
+      ATOMIC_DEC(&batch_record_count_);
+      if(OB_FAIL(resource_collector_->revert(record))) {
+        LOG_ERROR("resource collecter failed to revert record", K(record));
+      }
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "ObLogMinerBatchRecordWriter exit unexpected\n");
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecordWriter::get_total_record_count(int64_t &record_count)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_));
+  } else {
+    record_count = ATOMIC_LOAD(&record_count_);
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecordWriter::get_total_batch_record_count(int64_t &batch_record_count)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_));
+  } else {
+    batch_record_count = ATOMIC_LOAD(&batch_record_count_);
+  }
+
+  return ret;
+}
+
+void ObLogMinerBatchRecordWriter::run1()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_));
+  } else {
+    const int64_t META_FLUSH_INTERVAL = 1000L * 1000; // 1s
+    while (OB_SUCC(ret) && ! lib::Threads::has_set_stop()) {
+      if (OB_FAIL(proceed_checkpoint_())) {
+        LOG_ERROR("logminer batchrecord writer failed to proceed checkpoint");
+      }
+      ob_usleep(META_FLUSH_INTERVAL);
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "Checkpointer exit unexpected\n");
+  }
+}
+
+int ObLogMinerBatchRecordWriter::set_file_info_(ObLogMinerBatchRecord &record,
+    const int64_t seq_no)
+{
+  int ret = OB_SUCCESS;
+   if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_));
+  } else {
+    const char *unused_data = nullptr;
+    const int64_t record_max_commit_ts = record.get_progress_range().max_commit_ts_;
+    const int64_t cur_trans_record_count = record.get_cur_trans_record_count();
+    int64_t data_len = 0;
+    record.get_data(unused_data, data_len);
+
+    record.set_seq_no(seq_no);
+    record.set_file_id(current_file_id_);
+    record.set_file_offset(current_file_offset_);
+
+    if (0 >= cur_trans_record_count &&
+        data_manager_->reach_end_progress(record_max_commit_ts)) {
+      record.set_file_end();
+      LOG_TRACE("max commit ts of batch record exceed the end_progress and there is no "
+          "pending transaction", K(record_max_commit_ts), K(cur_trans_record_count));
+    }
+
+    // invariant: current_file_offset_ < FILE_SPLIT_THRESHOLD
+    ATOMIC_AAF(&current_file_offset_, data_len);
+
+    if (current_file_offset_ >= FILE_SPLIT_THRESHOLD) {
+      ATOMIC_INC(&current_file_id_);
+      ATOMIC_STORE(&current_file_offset_, 0);
+      record.set_file_end();
+    }
+
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecordWriter::generate_meta_flush_task_(const ObLogMinerBatchRecord &record)
+{
+  int ret = OB_SUCCESS;
+  BatchRecordMetaTask *task = reinterpret_cast<BatchRecordMetaTask*>(ob_malloc(
+      sizeof(BatchRecordMetaTask), "LogMnrMetaTask"));
+  if (OB_ISNULL(task)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("meta task is null, allocate memory failed", K(task));
+  } else {
+    task = new(task) BatchRecordMetaTask();
+    task->file_id_ = record.get_file_id();
+    task->is_file_complete_ = record.is_file_end();
+    task->last_trans_end_ts_ = record.get_last_trans_end_ts();
+    task->record_cnt_ = record.get_written_record_count();
+    LOG_TRACE("generated meta flush task", KPC(task), "seq_no", record.get_seq_no());
+    if (OB_FAIL(batch_record_sw_.set(record.get_seq_no(), task))) {
+      LOG_ERROR("set task into batch_record sliding window failed", K(record));
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerBatchRecordWriter::proceed_checkpoint_()
+{
+  int ret = OB_SUCCESS;
+  BatchRecordMetaTaskPopFunc func;
+  BatchRecordMetaTask *last_poped_task = nullptr, *curr_task = nullptr;
+  ObLogMinerCheckpoint ckpt;
+  bool popped = true;
+  constexpr int64_t PRINT_INTERVAL = 10L * 1000 * 1000;
+
+  while (OB_SUCC(ret) && popped && OB_SUCC(batch_record_sw_.pop(func, curr_task, popped))) {
+    // continuity check
+    if (popped && nullptr != curr_task) {
+      if (OB_INVALID_TIMESTAMP == ckpt.progress_ || ckpt.progress_ <= curr_task->last_trans_end_ts_) {
+        ckpt.progress_ = curr_task->last_trans_end_ts_;
+      } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("the progress of meta_flush task rollbacked", K(ckpt), KPC(curr_task));
+      }
+
+      if (-1 == ckpt.cur_file_id_ || ckpt.cur_file_id_ <= curr_task->file_id_) {
+        ckpt.cur_file_id_ = curr_task->file_id_;
+      } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("the file id of meta_flush task rollbacked", K(ckpt), KPC(curr_task));
+      }
+
+      if (curr_task->is_file_complete_) {
+        if (-1 == ckpt.max_file_id_ || ckpt.max_file_id_ == curr_task->file_id_ - 1) {
+          ckpt.max_file_id_ = curr_task->file_id_;
+        } else {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_ERROR("max file id of meta_flush task not expected", K(ckpt), KPC(curr_task));
+        }
+      }
+
+      if (OB_SUCC(ret) && OB_FAIL(data_manager_->increase_record_count(curr_task->record_cnt_))) {
+        LOG_ERROR("failed to increase record count", K(curr_task->record_cnt_));
+      }
+
+      LOG_TRACE("task poped", KPC(curr_task));
+      if (nullptr != last_poped_task) {
+        ob_free(last_poped_task);
+      }
+      // cannot access last_poped_task afterwards
+      last_poped_task = curr_task;
+    } else {
+      last_poped_task = nullptr;
+    }
+  }
+
+  if (OB_ENTRY_NOT_EXIST != ret) {
+    LOG_ERROR("failed to proceed checkpoint", K(curr_task));
+  } else {
+    ret = OB_SUCCESS;
+  }
+
+  if (OB_SUCC(ret)) {
+    if (nullptr != curr_task) {
+      ckpt.cur_file_id_ = curr_task->file_id_;
+      ckpt.progress_ = curr_task->last_trans_end_ts_;
+      ckpt.max_file_id_ = curr_task->is_file_complete_ ? curr_task->file_id_ : curr_task->file_id_ - 1;
+
+      if (OB_FAIL(file_manager_->write_checkpoint(ckpt))) {
+        LOG_ERROR("file_manager failed to write checkpoint", K(ckpt), K(curr_task));
+      } else if (OB_FAIL(data_manager_->update_output_progress(ckpt.progress_))) {
+        LOG_ERROR("failed to update output progress", K(ckpt));
+      } else {
+        if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) {
+          LOG_INFO("succeed to proceed checkpoint", K(ckpt), K(curr_task));
+        }
+      }
+
+      if (OB_SUCC(ret)) {
+        // cannot access curr_task afterwards
+        LOG_TRACE("proceed checkpoint", K(ckpt), KPC(curr_task));
+        ob_free(curr_task);
+      }
+    } else {
+      // only one thread
+      if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) {
+        LOG_INFO("try to proceed checkpoint but no batch record meta task", K(popped));
+      }
+    }
+  }
+
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_batch_record_writer.h

@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_BATCH_WRITER_H_
+#define OCEANBASE_LOG_MINER_RECORD_BATCH_WRITER_H_
+
+#include "lib/thread/ob_multi_fixed_queue_thread.h"
+#include "lib/container/ob_ext_ring_buffer.h"
+#include "ob_log_miner_file_manager.h"
+#include "ob_log_miner_error_handler.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerBatchRecord;
+class ILogMinerDataManager;
+class ILogMinerResourceCollector;
+class ILogMinerFileManager;
+
+class ILogMinerBatchRecordWriter
+{
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int push(ObLogMinerBatchRecord *record) = 0;
+  virtual int get_total_record_count(int64_t &record_count) = 0;
+  virtual int get_total_batch_record_count(int64_t &batch_record_count) = 0;
+};
+
+struct BatchRecordMetaTask
+{
+  BatchRecordMetaTask() {reset();}
+  ~BatchRecordMetaTask() { reset(); }
+  void reset() {
+    file_id_ = -1;
+    last_trans_end_ts_ = OB_INVALID_TIMESTAMP;
+    record_cnt_ = 0;
+    is_file_complete_ = false;
+  }
+
+  TO_STRING_KV(
+    K(file_id_),
+    K(last_trans_end_ts_),
+    K(record_cnt_),
+    K(is_file_complete_)
+  )
+
+  int64_t file_id_;
+  int64_t last_trans_end_ts_;
+  int64_t record_cnt_;
+  bool is_file_complete_;
+};
+
+struct BatchRecordMetaTaskPopFunc
+{
+  bool operator()(int64_t sn, BatchRecordMetaTask *task) {
+    UNUSED(sn);
+    return nullptr != task;
+  }
+};
+
+typedef common::ObMQThread<1, ILogMinerBatchRecordWriter> BatchWriterThreadPool;
+typedef ObExtendibleRingBuffer<BatchRecordMetaTask> BatchRecordSlidingWindow;
+class ObLogMinerBatchRecordWriter: public ILogMinerBatchRecordWriter, BatchWriterThreadPool, lib::ThreadPool
+{
+public:
+  static const int64_t BATCH_WRITER_THREAD_NUM = 1L;
+  static const int64_t BATCH_WRITER_QUEUE_SIZE = 100000L;
+  static const int64_t FILE_SPLIT_THRESHOLD = (64L << 20); // 64 MB
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int push(ObLogMinerBatchRecord *record);
+  virtual int get_total_record_count(int64_t &record_count);
+  virtual int get_total_batch_record_count(int64_t &batch_record_count);
+public:
+  ObLogMinerBatchRecordWriter();
+  ~ObLogMinerBatchRecordWriter() {
+    destroy();
+  }
+
+  virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag);
+
+  virtual void run1();
+
+  int init(ILogMinerDataManager *data_mgr,
+      ILogMinerResourceCollector *res_collector,
+      ILogMinerFileManager *file_mgr,
+      ILogMinerErrorHandler *err_handle);
+
+private:
+  int set_file_info_(ObLogMinerBatchRecord &record,
+      const int64_t seq_no);
+
+  int generate_meta_flush_task_(const ObLogMinerBatchRecord &record);
+
+  int proceed_checkpoint_();
+
+private:
+  bool is_inited_;
+  int64_t current_file_id_;
+  int64_t current_file_offset_;
+  BatchRecordSlidingWindow batch_record_sw_;
+
+  int64_t record_count_;
+  int64_t batch_record_count_;
+  ILogMinerDataManager *data_manager_;
+  ILogMinerResourceCollector *resource_collector_;
+  ILogMinerFileManager *file_manager_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_br.cpp

@@ -0,0 +1,79 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_binlog_record.h"
+#include "ob_log_miner_br.h"
+#include "lib/ob_define.h"
+#include "lib/oblog/ob_log.h"
+
+namespace oceanbase
+{
+
+namespace oblogminer
+{
+
+int ObLogMinerBR::init(libobcdc::IObCDCInstance *host,
+    ICDCRecord *br,
+    const lib::Worker::CompatMode mode,
+    const transaction::ObTransID &trans_id,
+    const uint64_t tenant_id,
+    const int32_t major_version)
+{
+  int ret = OB_SUCCESS;
+  libobcdc::ObLogBR *oblog_br = nullptr;
+  share::SCN commit_scn;
+  if (nullptr == br) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get null binlog record", K(br), K(tenant_id), K(major_version));
+  } else if (OB_INVALID_TENANT_ID == tenant_id) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid tenant_id", K(br), K(tenant_id), K(major_version));
+  } else if (OB_ISNULL(oblog_br = static_cast<libobcdc::ObLogBR*>(br->getUserData()))) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get ICDCRecord without user data, unexpected", KP(oblog_br));
+  } else if (OB_FAIL(commit_scn.convert_for_inner_table_field(oblog_br->get_commit_version()))) {
+    LOG_ERROR("failed to convert commit_version to scn", "commit_version", oblog_br->get_commit_version());
+  } else {
+    br_host_ = host;
+    br_ = br;
+    compat_mode_ = mode;
+    tenant_id_ = tenant_id;
+    major_version_ = major_version;
+    commit_scn_ = commit_scn;
+    record_type_ = static_cast<RecordType>(br->recordType());
+    trans_id_ = trans_id;
+  }
+  return ret;
+}
+
+void ObLogMinerBR::reset()
+{
+  is_filtered_ = false;
+  compat_mode_ = lib::Worker::CompatMode::INVALID;
+  seq_no_ = 0;
+  tenant_id_ = OB_INVALID_TENANT_ID;
+  major_version_ = 0;
+  commit_scn_.reset();
+  record_type_ = RecordType::EUNKNOWN;
+  trans_id_ = transaction::ObTransID();
+  if (nullptr != br_host_ && nullptr != br_) {
+    br_host_->release_record(br_);
+  }
+  br_host_ = nullptr;
+  br_ = nullptr;
+}
+
+}
+
+}

src/logservice/logminer/ob_log_miner_br.h

@@ -0,0 +1,136 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_BR_H_
+#define OCEANBASE_LOG_MINER_BR_H_
+
+#include "libobcdc.h"
+#include "lib/worker.h"
+#include "ob_log_miner_recyclable_task.h"
+#include "storage/tx/ob_trans_define.h"
+
+#ifndef OB_USE_DRCMSG
+#include "ob_cdc_msg_convert.h"
+#else
+#include <drcmsg/BR.h>               // ICDCRecord
+#include <drcmsg/MD.h>               // ITableMeta
+#include <drcmsg/MsgWrapper.h>       // IStrArray
+#include <drcmsg/binlogBuf.h>        // binlogBuf
+#include <drcmsg/DRCMessageFactory.h>
+#endif
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+class ObLogMinerBR: public ObLogMinerRecyclableTask
+{
+public:
+  ObLogMinerBR():
+      ObLogMinerRecyclableTask(TaskType::BINLOG_RECORD),
+      is_filtered_(false),
+      compat_mode_(lib::Worker::CompatMode::INVALID),
+      seq_no_(0),
+      tenant_id_(OB_INVALID_TENANT_ID),
+      major_version_(0),
+      commit_scn_(),
+      record_type_(EUNKNOWN),
+      trans_id_(),
+      br_host_(nullptr),
+      br_(nullptr)
+  {
+  }
+
+  ~ObLogMinerBR() {
+    destroy();
+  }
+
+  int init(libobcdc::IObCDCInstance *host,
+      ICDCRecord *br,
+      const lib::Worker::CompatMode mode,
+      const transaction::ObTransID &trans_id,
+      const uint64_t tenant_id,
+      const int32_t major_version);
+
+  // reentrant
+  void reset();
+
+  void destroy() {
+    reset();
+  }
+
+  ICDCRecord *get_br() {
+    return br_;
+  }
+
+  void mark_filtered() {
+    is_filtered_ = true;
+  }
+
+  bool is_filtered() const {
+    return is_filtered_;
+  }
+
+  lib::Worker::CompatMode get_compat_mode() const {
+    return compat_mode_;
+  }
+
+  uint64_t get_tenant_id() const {
+    return tenant_id_;
+  }
+
+  share::SCN get_commit_scn() const {
+    return commit_scn_;
+  }
+
+  void set_seq_no(int64_t seq_no) {
+    seq_no_ = seq_no;
+  }
+
+  int64_t get_seq_no() const {
+    return seq_no_;
+  }
+
+  RecordType get_record_type() const {
+    return record_type_;
+  }
+
+  const transaction::ObTransID& get_trans_id() const {
+    return trans_id_;
+  }
+
+  TO_STRING_KV(
+    K(is_filtered_),
+    K(compat_mode_),
+    K(seq_no_),
+    K(tenant_id_),
+    K(commit_scn_),
+    K(record_type_),
+    K(trans_id_)
+  );
+
+private:
+  bool is_filtered_;
+  lib::Worker::CompatMode compat_mode_;
+  int64_t seq_no_;
+  uint64_t tenant_id_;
+  int32_t major_version_;
+  share::SCN commit_scn_;
+  RecordType record_type_;
+  transaction::ObTransID trans_id_;
+  libobcdc::IObCDCInstance *br_host_;
+  ICDCRecord *br_;
+};
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_br_converter.cpp

@@ -0,0 +1,202 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_br_converter.h"
+#include "ob_log_miner_resource_collector.h"
+#include "ob_log_miner_data_manager.h"
+#include "ob_log_miner_analysis_writer.h"
+#include "ob_log_miner_record.h"
+#include "ob_log_miner_br.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+const int64_t ObLogMinerBRConverter::PUSH_BR_TIMEOUT_TIME = 1L * 1000 * 1000;
+const int64_t ObLogMinerBRConverter::BR_CONVERTER_THREAD_NUM = 1L;
+const int64_t ObLogMinerBRConverter::BR_CONVERTER_QUEUE_SIZE = 100000L;
+
+ObLogMinerBRConverter::ObLogMinerBRConverter():
+    is_inited_(false),
+    data_manager_(nullptr),
+    writer_(nullptr),
+    resource_collector_(nullptr),
+    err_handle_(nullptr) { }
+
+ObLogMinerBRConverter::~ObLogMinerBRConverter()
+{
+  destroy();
+}
+
+int ObLogMinerBRConverter::init(ILogMinerDataManager *data_manager,
+    ILogMinerAnalysisWriter *writer,
+    ILogMinerResourceCollector *resource_collector,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("br converter has already been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data_manager) || OB_ISNULL(writer) || OB_ISNULL(resource_collector)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get null components pointer", K(data_manager), K(writer), K(resource_collector));
+  } else if (OB_FAIL(BRConverterThreadPool::init(BR_CONVERTER_THREAD_NUM, BR_CONVERTER_QUEUE_SIZE))) {
+    LOG_ERROR("BRConverterThreads failed to init");
+  } else {
+    data_manager_ = data_manager;
+    writer_ = writer;
+    resource_collector_ = resource_collector;
+    err_handle_ = err_handle;
+    is_inited_ = true;
+    LOG_INFO("ObLogMinerBRConverter finished to init", K(data_manager_), K(writer_),
+        K(resource_collector_));
+    LOGMINER_STDOUT_V("ObLogMinerBRConverter finished to init\n");
+  }
+  return ret;
+}
+
+int ObLogMinerBRConverter::start()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("logminer br converter is not initialized", K(is_inited_));
+  } else if (OB_FAIL(BRConverterThreadPool::start())) {
+    LOG_ERROR("logminer br converter failed to start");
+  } else {
+    // start
+    LOG_INFO("ObLogMinerBRConverter starts");
+  }
+  return ret;
+}
+
+void ObLogMinerBRConverter::stop()
+{
+  BRConverterThreadPool::mark_stop_flag();
+  LOG_INFO("ObLogMinerBRConverter stopped");
+}
+
+void ObLogMinerBRConverter::wait()
+{
+  BRConverterThreadPool::stop();
+   LOG_INFO("ObLogMinerBRConverter finished to wait");
+}
+
+void ObLogMinerBRConverter::destroy()
+{
+  if (IS_INIT) {
+    data_manager_ = nullptr;
+    writer_ = nullptr;
+    resource_collector_ = nullptr;
+    err_handle_ = nullptr;
+    BRConverterThreadPool::destroy();
+    is_inited_ = false;
+    LOG_INFO("ObLogMinerBRConverter destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerBRConverter destroyed\n");
+  }
+}
+
+int ObLogMinerBRConverter::push(ObLogMinerBR *logminer_br)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRConverter is not inited", K(logminer_br));
+  } else if (OB_ISNULL(logminer_br)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get a null logminer br in br converter", K(logminer_br));
+  } else {
+    LOG_TRACE("BRConverter push logminer_br into queue", KPC(logminer_br));
+    int64_t hash_val = 0;
+    RETRY_FUNC(is_stoped(), *(static_cast<ObMQThread*>(this)), push, logminer_br, hash_val, PUSH_BR_TIMEOUT_TIME);
+    if (OB_SUCC(ret)) {
+      if (is_stoped()) {
+        ret = OB_IN_STOP_STATE;
+      }
+    } else {
+      LOG_ERROR("push logminer br into the queue of ObMQThread failed", K(logminer_br), K(hash_val));
+    }
+  }
+
+
+  return ret;
+}
+
+int ObLogMinerBRConverter::get_total_task_count(int64_t &record_count)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRConverter is not inited", K(this));
+  } else if (OB_FAIL(BRConverterThreadPool::get_total_task_num(record_count))) {
+    LOG_ERROR("failed to get record count");
+  }
+
+  return ret;
+}
+
+int ObLogMinerBRConverter::handle(void *data, const int64_t thread_index, volatile bool &stop_flag)
+{
+  int ret = OB_SUCCESS;
+  ObLogMinerBR *br = static_cast<ObLogMinerBR*>(data);
+  ObLogMinerRecord *record = nullptr;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRConverter is not inited", K(br), K(is_inited_));
+  } else if (OB_ISNULL(br)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get null br when handle br converter tasks", K(br));
+  } else if (br->is_filtered()) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("filtered br appears in converter", K(br));
+  } else if (OB_FAIL(data_manager_->get_logminer_record(record))) {
+    LOG_ERROR("get logminer record from data manager failed", K(record), K(data_manager_));
+  } else if (OB_ISNULL(record)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get a null record from data_manager", K(record), K(data_manager_));
+  } else if (OB_FAIL(generate_logminer_record_(*br, *record))) {
+    LOG_ERROR("failed to generate logminer record", K(br), K(record));
+  } else if (OB_FAIL(writer_->push(record))) {
+    LOG_ERROR("failed to push record into writer", K(record), K(writer_));
+  }
+  if (OB_SUCC(ret) && OB_NOT_NULL(br)) {
+    if (OB_FAIL(resource_collector_->revert(br))) {
+      LOG_ERROR("failed to revert logminer br", K(br), K(resource_collector_));
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "ObLogMinerBRFilter exit unexpected\n");
+  }
+  return ret;
+}
+
+int ObLogMinerBRConverter::generate_logminer_record_(ObLogMinerBR &br,
+    ObLogMinerRecord &record)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(record.init(br))) {
+    LOG_ERROR("record failed to init using logminerbr", K(br), K(record));
+  } else if (record.is_ddl_record() || record.is_dml_record()) {
+    if (OB_FAIL(record.build_stmts(br))) {
+      LOG_ERROR("failed to build stmts for record", K(record), K(br));
+    }
+  }
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_br_converter.h

@@ -0,0 +1,79 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_BR_CONVERTER_H_
+#define OCEANBASE_LOG_MINER_BR_CONVERTER_H_
+
+#include "lib/thread/ob_multi_fixed_queue_thread.h"
+#include "ob_log_miner_error_handler.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+class ObLogMinerBR;
+class ObLogMinerRecord;
+class ILogMinerDataManager;
+class ILogMinerResourceCollector;
+class ILogMinerAnalysisWriter;
+
+class ILogMinerBRConverter {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int push(ObLogMinerBR *logminer_br) = 0;
+  virtual int get_total_task_count(int64_t &record_count) = 0;
+};
+
+typedef common::ObMQThread<1, ILogMinerBRConverter> BRConverterThreadPool;
+
+class ObLogMinerBRConverter: public ILogMinerBRConverter, public BRConverterThreadPool
+{
+public:
+  static const int64_t BR_CONVERTER_THREAD_NUM;
+  static const int64_t BR_CONVERTER_QUEUE_SIZE;
+  static const int64_t PUSH_BR_TIMEOUT_TIME;
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int push(ObLogMinerBR *logminer_br);
+  virtual int get_total_task_count(int64_t &record_count);
+
+public:
+  ObLogMinerBRConverter();
+  ~ObLogMinerBRConverter();
+  int init(ILogMinerDataManager *data_manager,
+      ILogMinerAnalysisWriter *writer,
+      ILogMinerResourceCollector *resource_collector,
+      ILogMinerErrorHandler *err_handle_);
+
+  virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag);
+
+private:
+  int generate_logminer_record_(ObLogMinerBR &br, ObLogMinerRecord &record);
+
+private:
+  bool is_inited_;
+  ILogMinerDataManager *data_manager_;
+  ILogMinerAnalysisWriter *writer_;
+  ILogMinerResourceCollector *resource_collector_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_br_filter.cpp

@@ -0,0 +1,469 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_br_filter.h"
+#include "ob_log_miner_br.h"
+#include "ob_log_miner_br_producer.h"
+#include "ob_log_miner_br_converter.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+//////////////////////////////////// ColumnBRFilterPlugin ///////////////////////////////////
+
+ColumnBRFilterPlugin::ColumnBRFilterPlugin(ObIAllocator *alloc):
+    alloc_(alloc),
+    multi_table_column_cond_(alloc_)
+    { }
+
+int ColumnBRFilterPlugin::init(const char *table_cond_str)
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_FAIL(multi_table_column_cond_.init(table_cond_str))) {
+    LOG_ERROR("multi_table_cond failed to init", K(table_cond_str), K(alloc_));
+  }
+
+  return ret;
+}
+
+bool ColumnBRFilterPlugin::need_process_(const RecordType type)
+{
+  bool bret = false;
+  switch (type) {
+    case EINSERT:
+    case EUPDATE:
+    case EDELETE:
+      bret = true;
+      break;
+    default:
+      bret = false;
+      break;
+  }
+  return bret;
+}
+
+bool ColumnBRFilterPlugin::is_table_match_(const ObLogMinerTableColumnCond& table_column_cond,
+     const char *db_name,
+     const char *table_name)
+{
+  return (table_column_cond.db_name_.is_empty() || 0 == strcmp(table_column_cond.db_name_.ptr(), db_name)) &&
+         (table_column_cond.table_name_.is_empty() || 0 == strcmp(table_column_cond.table_name_.ptr(), table_name));
+}
+
+bool ColumnBRFilterPlugin::satisfy_column_cond_(
+     const ObLogMinerColumnVals &col_cond,
+     const binlogBuf *new_cols,
+     const int64_t new_col_cnt,
+     const binlogBuf *old_cols,
+     const int64_t old_col_cnt,
+     ITableMeta *tbl_meta)
+{
+  bool satisfy_new_col_cond_met = true;
+  bool satisfy_old_col_cond_met = true;
+  const int64_t col_cond_cnt = col_cond.count();
+  ARRAY_FOREACH_X(col_cond, col_val_idx, col_cond_cnt,
+      (satisfy_old_col_cond_met || satisfy_new_col_cond_met)) {
+    const ObLogMinerColVal &col_val = col_cond[col_val_idx];
+    int64_t column_idx = tbl_meta->getColIndex(col_val.col_.ptr());
+    if (0 > column_idx) {
+      satisfy_new_col_cond_met = false;
+      satisfy_old_col_cond_met = false;
+      LOG_TRACE("col in col_cond doesn't exist in the row", K(column_idx), K(col_val));
+    } else {
+      const char *new_data_ptr = nullptr, *old_data_ptr = nullptr;
+      int64_t new_data_len = 0, old_data_len = 0;
+      if (column_idx < new_col_cnt) {
+        new_data_ptr = new_cols[column_idx].buf;
+        new_data_len = new_cols[column_idx].buf_used_size;
+      } else {
+        // -1 indicates new_data don't exist
+        new_data_len = -1;
+      }
+      if (column_idx < old_col_cnt) {
+        old_data_ptr = old_cols[column_idx].buf;
+        old_data_len = old_cols[column_idx].buf_used_size;
+      } else {
+        // -1 indicates old_data don't exist
+        old_data_len = -1;
+      }
+
+      if (satisfy_new_col_cond_met) {
+        satisfy_new_col_cond_met = is_data_match_column_val_(new_data_ptr, new_data_len, col_val);
+      }
+      if (satisfy_old_col_cond_met) {
+        satisfy_old_col_cond_met = is_data_match_column_val_(old_data_ptr, old_data_len, col_val);
+      }
+    }
+  }
+
+  return satisfy_old_col_cond_met || satisfy_new_col_cond_met;
+}
+
+bool ColumnBRFilterPlugin::is_data_match_column_val_(const char *data_ptr,
+     const int64_t data_len,
+     const ObLogMinerColVal &col_val)
+{
+  bool bret = true;
+  if (data_len != col_val.val_.length()) {
+    bret = false;
+    LOG_TRACE("the length of new_data is not same as the length of new col_val", K(col_val),
+      K(data_ptr), K(data_len));
+  } else if (nullptr == data_ptr) {
+    if (col_val.is_null_) {
+      bret = true;
+      LOG_TRACE("data_ptr and the val in col_val are both null", K(col_val),
+        K(data_ptr));
+    } else {
+      bret = false;
+    }
+  } else {
+    if (0 != MEMCMP(data_ptr, col_val.val_.ptr(), min(data_len, col_val.val_.length()))) {
+      bret = false;
+      LOG_TRACE("new_data doesn't match", K(col_val),
+        K(data_ptr));
+    } else {
+      bret = true;
+      LOG_TRACE("new_data match successfully", K(col_val),
+        K(data_ptr));
+    }
+  }
+  return bret;
+}
+
+int ColumnBRFilterPlugin::filter(ObLogMinerBR &br, bool &need_filter)
+{
+  int ret = OB_SUCCESS;
+  need_filter = false;
+  ICDCRecord *cdc_br = br.get_br();
+  if (OB_ISNULL(cdc_br)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get a null cdc binlog record, unexpected", K(cdc_br));
+  } else if (!need_process_(static_cast<RecordType>(cdc_br->recordType()))) {
+    need_filter = false;
+    LOG_TRACE("doesn't filter some record needn't to be processed");
+  } else {
+    ITableMeta *tbl_meta = cdc_br->getTableMeta();
+    if (OB_ISNULL(tbl_meta)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("get null table meta when filter record", "recordType", cdc_br->recordType());
+    } else {
+      const char *tenant_db_name = cdc_br->dbname();
+      const char *tbl_name = cdc_br->tbname();
+      unsigned int new_col_cnt = 0, old_col_cnt = 0;
+      binlogBuf *new_cols = cdc_br->newCols(new_col_cnt);
+      binlogBuf *old_cols = cdc_br->oldCols(old_col_cnt);
+
+      // expect tenant_db_name as tenant_name.db_name
+      const char *db_name = nullptr;
+      if (OB_ISNULL(tenant_db_name) || OB_ISNULL(tbl_name)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("dbname or tblname is null", K(tenant_db_name), K(tbl_name));
+      } else if (0 == multi_table_column_cond_.table_column_conds_.count()) {
+        need_filter = false;
+      } else if (OB_ISNULL(db_name = STRCHR(tenant_db_name, '.') + 1)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("unexpected tenant_db_name format", K(tenant_db_name), K(tbl_name));
+      } else {
+        bool satisfy_one_cond = false;
+
+        DbAndTableWrapper db_and_table_wrapper(db_name, tbl_name);
+        ObLogMinerTableColumnCond *table_column_cond = nullptr;
+        if (OB_FAIL(multi_table_column_cond_.table_column_conds_.get(db_and_table_wrapper, table_column_cond))) {
+          if (OB_ENTRY_NOT_EXIST == ret) {
+            ret = OB_SUCCESS;
+          } else {
+            LOG_ERROR("get table column cond failed", K(db_and_table_wrapper));
+          }
+        } else if (nullptr != table_column_cond) {
+          const int64_t column_conds_cnt = table_column_cond->column_conds_.count();
+          ARRAY_FOREACH_X(table_column_cond->column_conds_, col_cond_idx, column_conds_cnt,
+              OB_SUCC(ret) && ! satisfy_one_cond) {
+            const ObLogMinerColumnVals &col_cond = table_column_cond->column_conds_[col_cond_idx];
+            satisfy_one_cond = satisfy_column_cond_(col_cond, new_cols,
+                new_col_cnt, old_cols, old_col_cnt, tbl_meta);
+          } // ARRAY_FOREACH column_conds
+
+          if (table_column_cond->column_conds_.empty()) {
+            satisfy_one_cond = true;
+          }
+        }
+        need_filter = !satisfy_one_cond;
+      } // else get legal arguments
+    }
+  }
+
+  return ret;
+}
+
+void ColumnBRFilterPlugin::destroy()
+{
+  multi_table_column_cond_.destroy();
+}
+
+//////////////////////////////////// OperationFilterPlugin ///////////////////////////////////
+OperationBRFilterPlugin::OperationBRFilterPlugin():
+    op_cond_() { }
+
+int OperationBRFilterPlugin::init(const char *op_cond_str)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(op_cond_.init(op_cond_str))) {
+    LOG_ERROR("op_cond init failed", K(op_cond_str));
+  }
+  return ret;
+}
+
+int OperationBRFilterPlugin::filter(ObLogMinerBR &br, bool &need_filter)
+{
+  int ret = OB_SUCCESS;
+  ICDCRecord *cdc_br = br.get_br();
+  need_filter = false;
+  if (OB_ISNULL(cdc_br)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("operation filter get a null cdc br", K(cdc_br));
+  } else {
+    need_filter = ! op_cond_.is_record_type_match(br.get_record_type());
+  }
+  return ret;
+}
+
+void OperationBRFilterPlugin::destroy()
+{
+  op_cond_.reset();
+}
+
+//////////////////////////////////// BRFilter ///////////////////////////////////
+
+const int64_t ObLogMinerBRFilter::BR_FILTER_THREAD_NUM = 1L;
+const int64_t ObLogMinerBRFilter::BR_FILTER_QUEUE_SIZE = 100000L;
+
+ObLogMinerBRFilter::ObLogMinerBRFilter():
+    is_inited_(false),
+    plugin_allocator("FilterPlugin"),
+    filter_pipeline_(),
+    data_manager_(nullptr),
+    resource_collector_(nullptr),
+    br_converter_(nullptr),
+    err_handle_(nullptr) { }
+
+ObLogMinerBRFilter::~ObLogMinerBRFilter()
+{
+  destroy();
+}
+
+int ObLogMinerBRFilter::init(const char *table_column_cond,
+    const char *op_cond,
+    ILogMinerDataManager *data_manager,
+    ILogMinerResourceCollector *resource_collector,
+    ILogMinerBRConverter *br_converter,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+  ColumnBRFilterPlugin *column_plugin = nullptr;
+  OperationBRFilterPlugin *op_plugin = nullptr;
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("oblogminer br_filter is already inited, no need to init again", K(is_inited_),
+        K(table_column_cond), K(op_cond), K(data_manager), K(resource_collector), K(br_converter));
+  } else if (OB_FAIL(BRFilterThreadPool::init(BR_FILTER_THREAD_NUM, BR_FILTER_QUEUE_SIZE))) {
+    LOG_ERROR("BRFilterThreadPool failed to init");
+  } else if (OB_ISNULL(column_plugin =
+      static_cast<ColumnBRFilterPlugin*>(plugin_allocator.alloc(sizeof(ColumnBRFilterPlugin))))) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("allocate memory for table filter plugin failed", K(column_plugin));
+  } else if (OB_ISNULL(op_plugin =
+      static_cast<OperationBRFilterPlugin*>(plugin_allocator.alloc(sizeof(OperationBRFilterPlugin))))) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("allocate memory for operation filter plugin failed", K(op_plugin));
+  } else {
+    column_plugin = new (column_plugin) ColumnBRFilterPlugin(&plugin_allocator);
+    op_plugin = new (op_plugin) OperationBRFilterPlugin();
+    if (OB_FAIL(column_plugin->init(table_column_cond))) {
+      LOG_ERROR("table plugin init failed", K(table_column_cond));
+    } else if (OB_FAIL(op_plugin->init(op_cond))) {
+      LOG_ERROR("op plugin init failed", K(op_cond));
+    } else if (OB_FAIL(filter_pipeline_.push_back(op_plugin))) {
+      LOG_ERROR("filter pipeline push back op_plugin failed", K(op_plugin), K(filter_pipeline_));
+    } else if (OB_FAIL(filter_pipeline_.push_back(column_plugin))) {
+      LOG_ERROR("filter pipeline push back column_plugin failed", K(column_plugin), K(filter_pipeline_));
+    } else {
+      data_manager_ = data_manager;
+      resource_collector_ = resource_collector;
+      br_converter_ = br_converter;
+      err_handle_ = err_handle;
+      is_inited_ = true;
+      LOG_INFO("ObLogMinerBRFilter finished to init", K(table_column_cond), K(op_cond));
+      LOGMINER_STDOUT_V("ObLogMinerBRFilter finished to init\n");
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerBRFilter::start()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRFilter hasn't been inited yet", K(is_inited_));
+  } else if (OB_FAIL(BRFilterThreadPool::start())) {
+    LOG_ERROR("BRFilterThreadPool failed to start");
+  } else {
+    LOG_INFO("ObLogMinerBRFilter starts successfully");
+  }
+  return ret;
+}
+
+void ObLogMinerBRFilter::stop()
+{
+  int ret = OB_SUCCESS;
+  if (IS_INIT) {
+    BRFilterThreadPool::mark_stop_flag();
+    LOG_INFO("ObLogMinerBRFilter stopped");
+  } else {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ObLogMinerBRFilter is not init or has been stopped");
+  }
+}
+
+void ObLogMinerBRFilter::wait()
+{
+  int ret = OB_SUCCESS;
+  if (IS_INIT) {
+    BRFilterThreadPool::stop();
+    LOG_INFO("ObLogMinerBRFilter finished to wait");
+  } else {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ObLogMinerBRFilter is not init or has been stopped");
+  }
+}
+
+void ObLogMinerBRFilter::destroy()
+{
+  if (IS_INIT) {
+    filter_pipeline_.destroy();
+    data_manager_ = nullptr;
+    resource_collector_ = nullptr;
+    br_converter_ = nullptr;
+    plugin_allocator.clear();
+    err_handle_ = nullptr;
+    BRFilterThreadPool::destroy();
+    is_inited_ = false;
+    LOG_INFO("ObLogMinerBRFilter destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerBRFilter destroyed\n");
+  }
+}
+
+int ObLogMinerBRFilter::push(ObLogMinerBR *logminer_br)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRFilter is not inited", K(this));
+  } else if (OB_ISNULL(logminer_br)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("ObLogMiner get a null logminer br", K(logminer_br));
+  } else {
+    LOG_TRACE("BRFilter start to process logminer_br", KPC(logminer_br));
+    uint64_t hash_val = logminer_br->get_seq_no(); // hash_val always equal to 0
+    constexpr int64_t PUSH_TIMEOUT_TIME = 1L * 1000 * 1000; // 1 sec
+    // TODO: caculate hash_val for multi-thread
+    RETRY_FUNC(is_stoped(), *(static_cast<ObMQThread*>(this)), push, logminer_br, hash_val, PUSH_TIMEOUT_TIME);
+
+    if (OB_FAIL(ret)) {
+      LOG_ERROR("push logminer br into logminer brfilter failed", K(logminer_br), K(hash_val));
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerBRFilter::get_total_task_count(int64_t &task_count)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRFilter is not inited", K(this));
+  } else if (OB_FAIL(BRFilterThreadPool::get_total_task_num(task_count))) {
+    LOG_ERROR("failed to get total task num");
+  }
+
+  return ret;
+}
+
+int ObLogMinerBRFilter::handle(void *data, const int64_t thread_index, volatile bool &stop_flag)
+{
+  int ret = OB_SUCCESS;
+  ObLogMinerBR *logminer_br = static_cast<ObLogMinerBR*>(data);
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRFilter is not inited", K(is_inited_), K(data));
+  } else if (OB_ISNULL(logminer_br)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("BRfilter got a null logminer br", K(logminer_br));
+  } else {
+    bool need_filter = false;
+    int tmp_ret = OB_SUCCESS;
+    if (OB_FAIL(filter_br_(*logminer_br, need_filter))) {
+      LOG_ERROR("ObLogMinerBRFilter failed to filter br", K(logminer_br), K(need_filter));
+    } else if (need_filter) {
+      logminer_br->mark_filtered();
+    } else {
+      // succ
+    }
+
+    if (OB_SUCC(ret)) {
+      if (logminer_br->is_filtered()) {
+        // the RecordType of filtered br must be EINSERT, EUPDATE or EDELETE
+        // records of the RecordType above don't impact the progress, i.e. analysis checkpoint.
+        LOG_TRACE("br has been filtered", KPC(logminer_br));
+        if (OB_FAIL(resource_collector_->revert(logminer_br))) {
+          LOG_ERROR("failed to revert logminer br", KPC(logminer_br), K(resource_collector_));
+        }
+      } else {
+        if (OB_FAIL(br_converter_->push(logminer_br))) {
+          LOG_ERROR("failed to push logminer br into br converter", K(br_converter_), KPC(logminer_br));
+        }
+      }
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "ObLogMinerBRFilter exit unexpected\n");
+  }
+  return ret;
+}
+
+int ObLogMinerBRFilter::filter_br_(ObLogMinerBR &br, bool &need_filter)
+{
+  int ret = OB_SUCCESS;
+  const int64_t filter_pipeline_size = filter_pipeline_.count();
+  need_filter = false;
+  ARRAY_FOREACH_X(filter_pipeline_, filter_idx, filter_pipeline_size, OB_SUCC(ret) && ! need_filter) {
+    IBRFilterPlugin *filter_plugin = filter_pipeline_.at(filter_idx);
+    if (OB_ISNULL(filter_plugin)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("get a null filter plugin in filter pipeline", K(filter_plugin));
+    } else if (OB_FAIL(filter_plugin->filter(br, need_filter))) {
+      LOG_ERROR("filter_plugin failed to filter br", K(filter_plugin), K(need_filter));
+    }
+  }
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_br_filter.h

@@ -0,0 +1,141 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_BR_FILTER_H_
+#define OCEANBASE_LOG_MINER_BR_FILTER_H_
+
+#include "lib/allocator/ob_concurrent_fifo_allocator.h"
+#include "lib/thread/ob_multi_fixed_queue_thread.h"
+#include "libobcdc.h"
+#include "ob_log_miner_filter_condition.h"
+#include "ob_log_miner_resource_collector.h"
+#include "lib/container/ob_se_array.h"
+#include "ob_log_miner_error_handler.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+class ObLogMinerBR;
+class ILogMinerBRProducer;
+class ILogMinerBRConverter;
+class ILogMinerDataManager;
+
+class IBRFilterPlugin
+{
+public:
+  virtual int filter(ObLogMinerBR &br, bool &need_filter) = 0;
+  DECLARE_PURE_VIRTUAL_TO_STRING;
+};
+
+class ColumnBRFilterPlugin: public IBRFilterPlugin
+{
+public:
+  virtual int filter(ObLogMinerBR &br, bool &need_filter) override;
+  TO_STRING_KV(K_(multi_table_column_cond));
+public:
+  ColumnBRFilterPlugin(ObIAllocator *alloc);
+  ~ColumnBRFilterPlugin() { destroy(); };
+  int init(const char *table_cond_str);
+  void destroy();
+
+
+private:
+  bool need_process_(const RecordType type);
+  bool is_table_match_(const ObLogMinerTableColumnCond& table_cond,
+       const char *db_name,
+       const char *table_name);
+  bool satisfy_column_cond_(const ObLogMinerColumnVals &col_cond,
+       const binlogBuf *new_cols,
+       const int64_t new_col_cnt,
+       const binlogBuf *old_cols,
+       const int64_t old_col_cnt,
+       ITableMeta *tbl_meta);
+  bool is_data_match_column_val_(const char *data_ptr,
+       const int64_t data_len,
+       const ObLogMinerColVal &col_val);
+
+private:
+  ObIAllocator *alloc_;
+  ObLogMinerMultiTableColumnCond multi_table_column_cond_;
+};
+
+class OperationBRFilterPlugin: public IBRFilterPlugin
+{
+public:
+  virtual int filter(ObLogMinerBR &br, bool &need_filter) override;
+  TO_STRING_KV(K_(op_cond));
+public:
+  OperationBRFilterPlugin();
+  ~OperationBRFilterPlugin() { destroy(); }
+  int init(const char *op_cond_str);
+  void destroy();
+
+private:
+  ObLogMinerOpCond op_cond_;
+};
+
+class ILogMinerBRFilter {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int push(ObLogMinerBR *logminer_br) = 0;
+  virtual int get_total_task_count(int64_t &task_count) = 0;
+};
+
+typedef ObMQThread<1, ILogMinerBRFilter> BRFilterThreadPool;
+
+class ObLogMinerBRFilter: public ILogMinerBRFilter, public BRFilterThreadPool
+{
+public:
+  static const int64_t BR_FILTER_THREAD_NUM;
+  static const int64_t BR_FILTER_QUEUE_SIZE;
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int push(ObLogMinerBR *logminer_br);
+  virtual int get_total_task_count(int64_t &task_count);
+
+public:
+  virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag);
+
+public:
+  ObLogMinerBRFilter();
+  ~ObLogMinerBRFilter();
+  int init(const char *table_cond,
+      const char *op_cond,
+      ILogMinerDataManager *data_manager,
+      ILogMinerResourceCollector *resource_collector,
+      ILogMinerBRConverter *br_converter,
+      ILogMinerErrorHandler *err_handle);
+
+private:
+  int filter_br_(ObLogMinerBR &br, bool &need_filter);
+
+private:
+  bool is_inited_;
+  ObArenaAllocator plugin_allocator;
+  common::ObSEArray<IBRFilterPlugin *, 4> filter_pipeline_;
+  ILogMinerDataManager *data_manager_;
+  ILogMinerResourceCollector *resource_collector_;
+  ILogMinerBRConverter *br_converter_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_br_producer.cpp

@@ -0,0 +1,330 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_br_producer.h"
+#include "ob_log_miner_br_filter.h"
+#include "ob_log_miner_args.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+const int64_t ObLogMinerBRProducer::BR_PRODUCER_THREAD_NUM = 1L;
+const int64_t ObLogMinerBRProducer::BR_PRODUCER_POOL_DEFAULT_NUM = 4L * 1024;
+
+class ObCdcConfigMapWrap {
+public:
+  explicit ObCdcConfigMapWrap(const std::map<std::string, std::string> &cdc_config):
+      cdc_config_(cdc_config) { }
+  int64_t to_string(char *buf, const int64_t buf_len) const
+  {
+    int64_t pos = 0;
+    FOREACH(it, cdc_config_) {
+      if (need_print_config(it->first)) {
+        databuff_print_kv(buf, buf_len, pos, it->first.c_str(), it->second.c_str());
+      }
+    }
+    return pos;
+  }
+
+  bool need_print_config(const std::string &config_key) const {
+    bool need_print = true;
+    if ((0 == config_key.compare("tenant_password"))
+        || (0 == config_key.compare("archive_dest"))) {
+      need_print = false;
+    }
+
+    return need_print;
+  }
+
+private:
+  const std::map<std::string, std::string> &cdc_config_;
+};
+
+ObLogMinerBRProducer::ObLogMinerBRProducer():
+    ThreadPool(BR_PRODUCER_THREAD_NUM),
+    is_inited_(false),
+    is_dispatch_end_(false),
+    start_time_us_(OB_INVALID_TIMESTAMP),
+    end_time_us_(OB_INVALID_TIMESTAMP),
+    curr_trans_id_(),
+    cdc_factory_(),
+    cdc_instance_(nullptr),
+    br_filter_(nullptr),
+    err_handle_(nullptr) { }
+
+int ObLogMinerBRProducer::init(const AnalyzerArgs &args,
+    ILogMinerBRFilter *filter,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+  std::map<std::string, std::string> cdc_config;
+
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("oblogminer binlogrecord producer init twice", K(is_inited_), K(args.start_time_us_));
+  } else if (OB_FAIL(build_cdc_config_map_(args, cdc_config))) {
+    LOG_ERROR("build cdc config map failed", K(args));
+  } else if (OB_ISNULL(cdc_instance_ = cdc_factory_.construct_obcdc())) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("alloc cdc_instance failed", K(cdc_instance_));
+  } else if (OB_ISNULL(br_filter_ = filter)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("filter for br producer is null, unexpected", K(br_filter_), K(filter));
+  } else if (OB_FAIL(cdc_instance_->init_with_start_tstamp_usec(cdc_config,
+      args.start_time_us_))) {
+    LOG_ERROR("init cdc instance failed", K(args.start_time_us_), K(ObCdcConfigMapWrap(cdc_config)));
+    LOGMINER_STDOUT("init cdc instance failed\n");
+  } else if (OB_FAIL(lib::ThreadPool::init())) {
+    LOG_ERROR("thread pool failed to init");
+  } else {
+    is_inited_ = true;
+    is_dispatch_end_ = false;
+    err_handle_ = err_handle;
+    start_time_us_ = args.start_time_us_;
+    end_time_us_ = args.end_time_us_;
+    LOG_INFO("ObLogMinerBRProducer finished to init", K(ObCdcConfigMapWrap(cdc_config)),
+        K(args.start_time_us_), K(filter));
+    LOGMINER_STDOUT_V("ObLogMinerBRProducer finished to init\n");
+  }
+  return ret;
+}
+
+void ObLogMinerBRProducer::run1()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerBRProducer not init", K(is_inited_), K(cdc_instance_));
+  } else if (OB_ISNULL(cdc_instance_) || OB_ISNULL(br_filter_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("cdc_instance is null in br producer", K(cdc_instance_), K(is_inited_), K(br_filter_));
+  } else {
+    bool next_dispatch_end = false;
+    const int64_t TIMEOUT_US = 1000L * 1000;
+    while (! has_set_stop() && OB_SUCC(ret)) {
+      ICDCRecord *br = nullptr;
+      ObLogMinerBR *logminer_br = nullptr;
+      uint64_t tenant_id = OB_INVALID_TENANT_ID;
+      int32_t major_version = 0;
+      lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID;
+      if (OB_FAIL(cdc_instance_->next_record(&br, major_version, tenant_id,TIMEOUT_US))) {
+        if (OB_TIMEOUT != ret && OB_IN_STOP_STATE != ret) {
+          LOG_ERROR("get record from cdc_instance failed", K(cdc_instance_));
+          LOGMINER_STDOUT_V("get record from cdc_instance failed\n");
+        }
+      } else if (OB_ISNULL(br)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("get a null ICDCRecord, unexpected", K(br), K(major_version), K(tenant_id));
+      } else if (OB_INVALID_TENANT_ID == tenant_id) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("get an invaild tenant_id", K(br), K(tenant_id), K(major_version));
+      } else if (OB_FAIL(get_tenant_compat_mode_(static_cast<RecordType>(br->recordType()),
+          tenant_id, compat_mode))) {
+        LOG_ERROR("get tenant compat mode failed", K(tenant_id), K(compat_mode));
+      } else if (lib::Worker::CompatMode::INVALID == compat_mode) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("get an invalid compat mode", K(compat_mode), K(tenant_id));
+      } else if (OB_FAIL(set_current_trans_id_(br))) {
+        LOG_ERROR("failed to set current trans_id", KP(br));
+      } else if (OB_ISNULL(logminer_br = reinterpret_cast<ObLogMinerBR*>(
+            ob_malloc(sizeof(ObLogMinerBR), "LogMnrBR")))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_ERROR("alloc logminer_br failed", K(logminer_br));
+      } else {
+        logminer_br = new(logminer_br) ObLogMinerBR();
+        if (OB_FAIL(logminer_br->init(cdc_instance_, br, compat_mode,
+            curr_trans_id_, tenant_id, major_version))) {
+          LOG_ERROR("logminer br failed to init", K(br), K(tenant_id), K(major_version));
+        } else if (end_time_us_ < logminer_br->get_commit_scn().convert_to_ts()) {
+          // don't let any logminer_br whose commit_ts is larger than end_time_us go through
+          if (is_dispatch_end_) {
+            LOG_TRACE("destroy any br whose commit_ts larger than end_time", "commit_ts_us",
+                logminer_br->get_commit_scn(), K(end_time_us_));
+            logminer_br->destroy();
+            ob_free(logminer_br);
+          } else if (is_trans_end_record_type(logminer_br->get_record_type())) {
+            next_dispatch_end = true;
+          } else {
+            // still dispatch
+          }
+        }
+      }
+
+      if (OB_SUCC(ret) && ! is_dispatch_end_) {
+        if (OB_FAIL(br_filter_->push(logminer_br))) {
+          LOG_ERROR("push logminer_br into br_filter failed", K(logminer_br), K(br_filter_));
+        } else {
+          is_dispatch_end_ = next_dispatch_end;
+        }
+      }
+
+      if (OB_TIMEOUT == ret) {
+        ret = OB_SUCCESS;
+      } else if (OB_IN_STOP_STATE == ret) {
+        LOG_INFO("cdc instance is stopped, br producer thread exits", K(cdc_instance_));
+      }
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "ObLogMinerBRProducer exit unexpected\n");
+  }
+}
+
+int ObLogMinerBRProducer::start()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("br producer is not inited", K(cdc_instance_), K(br_filter_));
+  } else if (OB_FAIL(cdc_instance_->launch())) {
+    LOG_ERROR("cdc instance failed to start");
+  } else if (OB_FAIL(lib::ThreadPool::start())) {
+    LOG_ERROR("thread pool for br producer failed to start");
+  } else {
+    LOG_INFO("ObLogMinerBRProducer starts");
+  }
+
+  return ret;
+}
+
+void ObLogMinerBRProducer::stop()
+{
+  lib::ThreadPool::stop();
+  LOG_INFO("ObLogMinerBRProducer stopped");
+}
+
+void ObLogMinerBRProducer::wait()
+{
+  lib::ThreadPool::wait();
+  cdc_instance_->stop();
+  LOG_INFO("ObLogMinerBRProducer finished to wait");
+}
+
+void ObLogMinerBRProducer::destroy()
+{
+  if (IS_INIT) {
+    ThreadPool::destroy();
+    cdc_factory_.deconstruct(cdc_instance_);
+    cdc_instance_ = nullptr;
+    br_filter_ = nullptr;
+    err_handle_ = nullptr;
+    curr_trans_id_.reset();
+    is_inited_ = false;
+    LOG_INFO("ObLogMinerBRProducer destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerBRProducer destroyed\n");
+  }
+}
+
+int ObLogMinerBRProducer::build_cdc_config_map_(const AnalyzerArgs &args,
+    std::map<std::string, std::string> &cdc_config)
+{
+  int ret = OB_SUCCESS;
+  const char *REFRESH_MODE = "meta_data_refresh_mode";
+  const char *FETCHING_MODE = "fetching_log_mode";
+  const char *TENANT_ENDPOINT = "tenant_endpoint";
+  const char *TENANT_USER = "tenant_user";
+  const char *TENANT_PASSWORD = "tenant_password";
+  const char *ARCHIVE_DEST = "archive_dest";
+  const char *LOG_LEVEL = "log_level";
+  const char *MEMORY_LIMIT = "memory_limit";
+  const char *REDO_DISPATCH_LIMIT = "redo_dispatcher_memory_limit";
+  const char *SQL_OUTPUT_ORDER = "enable_output_trans_order_by_sql_operation";
+  const char *TB_WHITE_LIST = "tb_white_list";
+
+  try {
+    cdc_config[LOG_LEVEL] = args.log_level_;
+    cdc_config[REFRESH_MODE] = "data_dict";
+    cdc_config[MEMORY_LIMIT] = "10G";
+    cdc_config[REDO_DISPATCH_LIMIT] = "128M";
+    cdc_config[SQL_OUTPUT_ORDER] = "1";
+    cdc_config[TB_WHITE_LIST] = args.table_list_;
+    if (nullptr != args.archive_dest_) {
+      cdc_config[FETCHING_MODE] = "direct";
+      cdc_config[ARCHIVE_DEST] = args.archive_dest_;
+    } else {
+      cdc_config[TENANT_ENDPOINT] = args.cluster_addr_;
+      cdc_config[TENANT_USER] = args.user_name_;
+      cdc_config[TENANT_PASSWORD] = args.password_;
+    }
+  } catch (std::exception &e) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("build cdc config map failed, exception occurs", "exception_info", e.what());
+  }
+
+  return ret;
+}
+
+int ObLogMinerBRProducer::get_tenant_compat_mode_(const RecordType type,
+    const uint64_t tenant_id,
+    lib::Worker::CompatMode &mode)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("br producer has not been initialized yet", K(is_inited_), K(cdc_instance_));
+  } else {
+    libobcdc::ObLogInstance *oblog_instance = static_cast<libobcdc::ObLogInstance*>(cdc_instance_);
+    if (HEARTBEAT == type) {
+      // in OBCDC, tenant_id is invalid for the HEARTBEAT type,
+      // and LogMiner won't deal with CompatMode for HEARTBEAT Record.
+      mode = lib::Worker::CompatMode::MYSQL;
+    } else if (OB_FAIL(oblog_instance->get_tenant_compat_mode(tenant_id, mode))) {
+      LOG_ERROR("get tenant compat mode failed", K(oblog_instance), K(tenant_id));
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerBRProducer::set_current_trans_id_(ICDCRecord *cdc_rec)
+{
+  int ret = OB_SUCCESS;
+  ObString cdc_trans_id_str;
+  uint filter_val_cnt = 0;
+  BinlogRecordImpl *br_impl = static_cast<BinlogRecordImpl*>(cdc_rec);
+  const binlogBuf *vals = br_impl->filterValues(filter_val_cnt);
+  const RecordType type = static_cast<RecordType>(cdc_rec->recordType());
+  if (EBEGIN == type) {
+    if (OB_ISNULL(vals) || filter_val_cnt <= 2) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("trans_id values is null for begin record", KP(vals), K(filter_val_cnt));
+    } else if (filter_val_cnt > 2) {
+      int64_t data = 0, pos = 0;
+      ObString start_str;
+      cdc_trans_id_str.assign_ptr(vals[1].buf, vals[1].buf_used_size);
+      start_str = cdc_trans_id_str.split_on('_');
+      if (OB_FAIL(parse_int(cdc_trans_id_str.ptr(), cdc_trans_id_str.length(), pos, data))) {
+        LOG_ERROR("failed to parse into for trans_id", K(start_str), K(cdc_trans_id_str));
+      } else {
+        curr_trans_id_ = data;
+        LOG_TRACE("parse trans_id for EBEGIN", K(start_str), K(cdc_trans_id_str), K(data));
+      }
+    } else {
+      LOG_TRACE("defensive branch, do not set trans_id for EBEGIN whose vals is NULL", KP(vals));
+    }
+  } else if (HEARTBEAT == type || EDDL == type) {
+    curr_trans_id_.reset();
+    LOG_TRACE("reset trans_id for type HEARTBEAT and EDDL", K(type));
+  } else {
+    // do not process other types
+    LOG_TRACE("ignore set current trans_id for type", K(type));
+  }
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_br_producer.h

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_BR_PRODUCER_H_
+#define OCEANBASE_LOG_MINER_BR_PRODUCER_H_
+
+#include "ob_log_miner_br.h"
+#include "ob_log_instance.h"
+#include "ob_log_miner_error_handler.h"
+#include "ob_log_miner_args.h"
+#include <map>
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerBRFilter;
+
+class ILogMinerBRProducer
+{
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+};
+
+class ObLogMinerBRProducer :public ILogMinerBRProducer, public lib::ThreadPool
+{
+public:
+  static const int64_t BR_PRODUCER_THREAD_NUM;
+  static const int64_t BR_PRODUCER_POOL_DEFAULT_NUM;
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  // need to be idempotent
+  virtual void destroy();
+
+private:
+  virtual void run1() override;
+
+public:
+  ObLogMinerBRProducer();
+  ~ObLogMinerBRProducer()
+  { destroy(); }
+  int init(const AnalyzerArgs &args,
+      ILogMinerBRFilter *filter,
+      ILogMinerErrorHandler *err_handle);
+
+private:
+  int build_cdc_config_map_(const AnalyzerArgs &args,
+      std::map<std::string, std::string> &cdc_config);
+  int get_tenant_compat_mode_(const RecordType type,
+      const uint64_t tenant_id,
+      lib::Worker::CompatMode &mode);
+
+  int set_current_trans_id_(ICDCRecord *cdc_rec);
+private:
+  bool is_inited_;
+  bool is_dispatch_end_;
+  int64_t start_time_us_;
+  int64_t end_time_us_;
+  transaction::ObTransID curr_trans_id_;
+  libobcdc::ObCDCFactory cdc_factory_;
+  libobcdc::IObCDCInstance *cdc_instance_;
+  ILogMinerBRFilter *br_filter_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_data_manager.cpp

@@ -0,0 +1,300 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_record_converter.h"
+#include "ob_log_miner_batch_record.h"
+#include "ob_log_miner_data_manager.h"
+#include "ob_log_miner_record.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+const int64_t ObLogMinerDataManager::THREAD_NUM = 1;
+
+ObLogMinerDataManager::ObLogMinerDataManager():
+    lib::ThreadPool(THREAD_NUM),
+    is_inited_(false),
+    stat_(),
+    mode_(LogMinerMode::UNKNOWN),
+    format_(RecordFileFormat::INVALID),
+    logminer_record_alloc_(),
+    start_progress_(OB_INVALID_TIMESTAMP),
+    end_progress_(OB_INVALID_TIMESTAMP),
+    output_progress_(OB_INVALID_TIMESTAMP),
+    record_count_(0),
+    err_handle_(nullptr) { }
+
+ObLogMinerDataManager::~ObLogMinerDataManager()
+{
+  destroy();
+}
+
+int ObLogMinerDataManager::init(const LogMinerMode mode,
+    const RecordFileFormat format,
+    const int64_t start_progress,
+    const int64_t end_progress,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("ObLogMinerDataManager has been initialized", K(is_inited_));
+  } else if (OB_FAIL(logminer_record_alloc_.init(OB_MALLOC_NORMAL_BLOCK_SIZE, "MinerRecAlloc",
+      OB_SERVER_TENANT_ID, LOGMINER_RECORD_LIMIT))) {
+    LOG_ERROR("allocator for logminer record failed to init");
+  } else if (OB_FAIL(logminer_batch_record_alloc_.init(OB_MALLOC_NORMAL_BLOCK_SIZE, "BatchRecAlloc",
+      OB_SERVER_TENANT_ID, LOGMINER_RECORD_LIMIT))) {
+    LOG_ERROR("allocator for logminer batch record failed to init");
+  } else if (OB_FAIL(lib::ThreadPool::init())) {
+    LOG_ERROR("ThreadPool for data manager failed to init");
+  } else {
+    stat_.reset();
+    mode_ = mode;
+    format_ = format;
+    start_progress_ = start_progress;
+    end_progress_ = end_progress;
+    err_handle_ = err_handle;
+    is_inited_ = true;
+    LOG_INFO("ObLogMinerDataManager finished to init", K(mode), K(format), K(start_progress),
+        K(end_progress));
+    LOGMINER_STDOUT_V("ObLogMinerDataManager finished to init\n");
+  }
+  return ret;
+}
+
+int ObLogMinerDataManager::start()
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("data manager hasn't been initialized", K(is_inited_));
+  } else if (OB_FAIL(lib::ThreadPool::start())) {
+    LOG_ERROR("thread pool for data manager failed to start");
+  } else {
+    LOG_INFO("ObLogMinerDataManager starts");
+  }
+
+  return ret;
+}
+
+void ObLogMinerDataManager::stop()
+{
+  lib::ThreadPool::stop();
+  LOG_INFO("ObLogMinerDataManager stopped");
+}
+
+void ObLogMinerDataManager::wait()
+{
+  lib::ThreadPool::wait();
+  LOG_INFO("ObLogMinerDataManager finished to wait");
+}
+
+void ObLogMinerDataManager::destroy()
+{
+  if (IS_INIT) {
+    is_inited_ = false;
+    stat_.reset();
+    mode_ = LogMinerMode::UNKNOWN;
+    format_ = RecordFileFormat::INVALID;
+    start_progress_ = OB_INVALID_TIMESTAMP;
+    end_progress_ = OB_INVALID_TIMESTAMP;
+    lib::ThreadPool::destroy();
+    logminer_batch_record_alloc_.destroy();
+    logminer_record_alloc_.destroy();
+    err_handle_ = nullptr;
+    LOG_INFO("ObLogMinerDataManager destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerDataManager destroyed\n");
+  }
+}
+
+int ObLogMinerDataManager::get_logminer_record(ObLogMinerRecord *&logminer_rec)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("get logminer record faild, data manager is not inited", K(is_inited_));
+  } else {
+    if (OB_ISNULL(logminer_rec = op_alloc_args(ObLogMinerRecord, &logminer_record_alloc_))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_ERROR("logminer record pool alloc record failed", K(logminer_rec));
+    } else {
+      ATOMIC_INC(&stat_.record_alloc_count_);
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerDataManager::release_log_miner_record(ObLogMinerRecord *logminer_rec)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("release logminer record faield, data manager is not inited", K(is_inited_));
+  } else {
+    logminer_rec->destroy();
+    op_free(logminer_rec);
+    ATOMIC_INC(&stat_.record_release_count_);
+  }
+  return ret;
+}
+
+int ObLogMinerDataManager::get_logminer_batch_record(ObLogMinerBatchRecord *&logminer_batch_rec)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("get logminer record faield, data manager is not inited", K(is_inited_));
+  } else {
+    ILogMinerRecordConverter *converter = ILogMinerRecordConverter::get_converter_instance(format_);
+    if (OB_ISNULL(converter)) {
+      ret = OB_NOT_SUPPORTED;
+      LOG_ERROR("logminer doesn't support the output format", K(format_));
+    } else if (OB_ISNULL(logminer_batch_rec = op_alloc_args(ObLogMinerBatchRecord,
+        &logminer_batch_record_alloc_, converter))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_ERROR("logminer record pool alloc record failed", K(logminer_batch_rec));
+    } else {
+      ATOMIC_INC(&stat_.batch_record_alloc_count_);
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerDataManager::release_logminer_batch_record(ObLogMinerBatchRecord *logminer_batch_rec)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("get logminer record faield, data manager is not inited", K(is_inited_));
+  } else if (OB_ISNULL(logminer_batch_rec)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("cannot release a null logminer batch rec", K(logminer_batch_rec));
+  } else {
+    logminer_batch_rec->reset();
+    op_free(logminer_batch_rec);
+    ATOMIC_INC(&stat_.batch_record_release_count_);
+  }
+  return ret;
+}
+
+int ObLogMinerDataManager::update_output_progress(const int64_t timestamp)
+{
+  int ret = OB_SUCCESS;
+  if (is_analysis_mode(mode_)) {
+    if (timestamp >= output_progress_) {
+      output_progress_ = timestamp;
+    } else {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("output progress rollbacked", K(timestamp), K(output_progress_), K(mode_));
+    }
+  } else if (is_flashback_mode(mode_)) {
+    if (timestamp <= output_progress_) {
+      output_progress_ = timestamp;
+    } else {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("output progress rollbacked", K(timestamp), K(output_progress_), K(mode_));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (reach_end_progress(timestamp)) {
+      if (OB_FAIL(LOGMINER_LOGGER.log_progress(record_count_, output_progress_, start_progress_, end_progress_))) {
+        LOG_WARN("log progress failed", K(record_count_), K(output_progress_), K(start_progress_), K(end_progress_));
+      }
+      LOGMINER_STDOUT("\nObLogMinerAnalyzer completed process\n");
+      err_handle_->handle_error(ret, "output progress reach end_progress");
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerDataManager::increase_record_count(int64_t count) {
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("data manager is not inited", K(is_inited_));
+  } else if (count < 0) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("increase negative count", K(count));
+  } else {
+    record_count_ += count;
+  }
+  return ret;
+}
+
+bool ObLogMinerDataManager::reach_end_progress(const int64_t progress) const
+{
+  bool bret = false;
+  if (OB_INVALID_TIMESTAMP == progress) {
+    bret = false;
+    LOG_WARN_RET(OB_ERR_UNEXPECTED, "get invalid progress when judging reach end progress",
+        K(progress), K(end_progress_));
+  } else if (is_analysis_mode(mode_)) {
+    bret = progress > end_progress_;
+  } else if (is_flashback_mode(mode_)) {
+    bret = progress < end_progress_;
+  }
+  return bret;
+}
+
+void ObLogMinerDataManager::run1()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("data manager not init");
+  } else {
+    const int64_t STAT_INTERVAL = 1L * 1000 * 1000; // 1s
+    const int64_t LOOP_INTERVAL = 100L * 1000; // 100 ms
+    while (OB_SUCC(ret) && !has_set_stop()) {
+      if (REACH_TIME_INTERVAL(STAT_INTERVAL)) {
+        do_statistics_();
+      }
+      ob_usleep(LOOP_INTERVAL);
+    }
+  }
+}
+
+void ObLogMinerDataManager::do_statistics_()
+{
+  static LogMinerDataManagerStat last_stat;
+  int ret = OB_SUCCESS;
+  LogMinerDataManagerStat cur_stat = stat_;
+  LogMinerDataManagerStat delta;
+  delta.record_alloc_count_ = cur_stat.record_alloc_count_ - last_stat.record_alloc_count_;
+  delta.record_release_count_ = cur_stat.record_release_count_ - last_stat.record_release_count_;
+  delta.batch_record_alloc_count_ = cur_stat.batch_record_alloc_count_ - last_stat.batch_record_alloc_count_;
+  delta.batch_record_alloc_count_ = cur_stat.batch_record_alloc_count_ - last_stat.batch_record_alloc_count_;
+  LOG_INFO("[DATA_MANAGER] [STAT] [RECORD_COUNT]", K(cur_stat), K(last_stat), K(delta),
+      "active_record_cnt", cur_stat.record_alloc_count_ - cur_stat.record_release_count_,
+      "active_batch_record_cnt", cur_stat.batch_record_alloc_count_ - cur_stat.batch_record_release_count_);
+  LOG_INFO("[DATA_MANAGER] [STAT] [OUTPUT_PROGRESS]", K(output_progress_), K(start_progress_), K(end_progress_),
+      K(record_count_));
+  // only print intermediate progress here.
+  if (output_progress_ != OB_INVALID_TIMESTAMP && !reach_end_progress(output_progress_)) {
+    if (OB_FAIL(LOGMINER_LOGGER.log_progress(record_count_, output_progress_, start_progress_, end_progress_))) {
+        LOG_WARN("log progress failed", K(record_count_), K(output_progress_), K(start_progress_), K(end_progress_));
+      }
+  }
+  last_stat = cur_stat;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_data_manager.h

@@ -0,0 +1,144 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_DATA_MANAGER_H_
+#define OCEANBASE_LOG_MINER_DATA_MANAGER_H_
+
+#include "lib/allocator/ob_concurrent_fifo_allocator.h"
+#include "lib/objectpool/ob_small_obj_pool.h"
+#include "lib/thread/threads.h"
+#include "libobcdc.h"
+#include "ob_log_miner_error_handler.h"
+#include "ob_log_miner_mode.h"
+
+#include "ob_log_miner_file_manager.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerBR;
+class ObLogMinerRecord;
+class ObLogMinerBatchRecord;
+
+class ILogMinerDataManager {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int get_logminer_record(ObLogMinerRecord *&logminer_rec) = 0;
+  virtual int release_log_miner_record(ObLogMinerRecord *logminer_rec) = 0;
+  virtual int get_logminer_batch_record(ObLogMinerBatchRecord *&logminer_batch_rec) = 0;
+  virtual int release_logminer_batch_record(ObLogMinerBatchRecord *logminer_batch_rec) = 0;
+  virtual int update_output_progress(const int64_t timestamp) = 0;
+  virtual int64_t get_output_progress() const = 0;
+  virtual int64_t get_record_count() const = 0;
+  virtual int increase_record_count(int64_t count) = 0;
+  virtual bool reach_end_progress(const int64_t progress) const = 0;
+};
+
+class LogMinerDataManagerStat
+{
+public:
+  LogMinerDataManagerStat() { reset(); }
+  ~LogMinerDataManagerStat() { reset(); }
+
+  void reset() {
+    record_alloc_count_ = 0;
+    record_release_count_ = 0;
+    batch_record_alloc_count_ = 0;
+    batch_record_release_count_ = 0;
+  }
+
+  // not atomic, just an estimate
+  LogMinerDataManagerStat &operator==(const LogMinerDataManagerStat &that) {
+    record_alloc_count_ = that.record_alloc_count_;
+    record_release_count_ = that.record_release_count_;
+    batch_record_alloc_count_ = that.batch_record_alloc_count_;
+    batch_record_release_count_ = that.batch_record_release_count_;
+    return *this;
+  }
+
+  TO_STRING_KV(
+    K(record_alloc_count_),
+    K(record_release_count_),
+    K(batch_record_alloc_count_),
+    K(batch_record_release_count_)
+  );
+
+  int64_t record_alloc_count_ CACHE_ALIGNED;
+  int64_t record_release_count_ CACHE_ALIGNED;
+  int64_t batch_record_alloc_count_ CACHE_ALIGNED;
+  int64_t batch_record_release_count_ CACHE_ALIGNED;
+};
+
+class ObLogMinerDataManager: public ILogMinerDataManager, public lib::ThreadPool
+{
+  static const int64_t THREAD_NUM;
+  static const int64_t LOGMINER_RECORD_LIMIT = 2 * 1024L * 1024 * 1024; // 2G
+  static const int64_t BATCH_RECORD_LIMIT = 2 * 1024L * 1024 * 1024; // 2G
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int get_logminer_record(ObLogMinerRecord *&logminer_rec);
+  virtual int release_log_miner_record(ObLogMinerRecord *logminer_rec);
+  virtual int get_logminer_batch_record(ObLogMinerBatchRecord *&logminer_batch_rec);
+  virtual int release_logminer_batch_record(ObLogMinerBatchRecord *logminer_batch_rec);
+  virtual int update_output_progress(const int64_t timestamp);
+  virtual int64_t get_output_progress() const
+  {
+    return output_progress_;
+  }
+  virtual int64_t get_record_count() const
+  {
+    return record_count_;
+  }
+  virtual int increase_record_count(int64_t count);
+  virtual bool reach_end_progress(const int64_t progress) const;
+
+public:
+  virtual void run1();
+
+public:
+  ObLogMinerDataManager();
+  ~ObLogMinerDataManager();
+  int init(const LogMinerMode mode,
+      const RecordFileFormat format,
+      const int64_t start_progress,
+      const int64_t end_progress,
+      ILogMinerErrorHandler *err_handle);
+
+private:
+  void do_statistics_();
+
+private:
+  bool is_inited_;
+  LogMinerDataManagerStat stat_;
+  LogMinerMode mode_;
+  RecordFileFormat format_;
+  common::ObConcurrentFIFOAllocator logminer_record_alloc_;
+  common::ObConcurrentFIFOAllocator logminer_batch_record_alloc_;
+  int64_t start_progress_;
+  int64_t end_progress_;
+  int64_t output_progress_;
+  int64_t record_count_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_error_handler.h

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_ERROR_HANDLER_H_
+#define OCEANBASE_LOG_MINER_ERROR_HANDLER_H_
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerErrorHandler
+{
+public:
+  virtual void handle_error(int err_code, const char *fmt, ...) = 0;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_file_index.cpp

@@ -0,0 +1,208 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_file_index.h"
+#include "logservice/logminer/ob_log_miner_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+////////////////////////////// FileIndexItem //////////////////////////////
+
+DEFINE_SERIALIZE(FileIndexItem)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf ,buf_len, pos, "%ld:%ld:%ld\n",
+      file_id_, range_.min_commit_ts_, range_.max_commit_ts_))) {
+    LOG_ERROR("failed to filled data into buf when serializing file index item", KPC(this));
+  }
+  return ret;
+}
+
+DEFINE_DESERIALIZE(FileIndexItem)
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_FAIL(parse_int(buf, data_len, pos, file_id_))) {
+    LOG_ERROR("failed to parse file_id", K(data_len), K(pos));
+  } else if (OB_FAIL(expect_token(buf, data_len, pos, ":"))) {
+    LOG_ERROR("failed tot get token ':'", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_int(buf, data_len, pos, range_.min_commit_ts_))) {
+    LOG_ERROR("failed to parse min_commit_ts", K(data_len), K(pos));
+  } else if (OB_FAIL(expect_token(buf, data_len, pos, ":"))) {
+    LOG_ERROR("failed tot get token ':'", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_int(buf, data_len, pos, range_.max_commit_ts_))) {
+    LOG_ERROR("failed to parse max_commit_ts", K(data_len), K(pos));
+  } else if (OB_FAIL(expect_token(buf, data_len, pos, "\n"))) {
+    LOG_ERROR("failed tot get token '\\n'", K(data_len), K(pos));
+  }
+
+  return ret;
+}
+
+DEFINE_GET_SERIALIZE_SIZE(FileIndexItem)
+{
+  int64_t size = 0;
+  const int64_t max_digit_len = 30;
+  char digit_buf[max_digit_len];
+  size += snprintf(digit_buf, max_digit_len, "%ld", file_id_);
+  size += 1;
+  size += snprintf(digit_buf, max_digit_len, "%ld", range_.min_commit_ts_);
+  size += 1;
+  size += snprintf(digit_buf, max_digit_len, "%ld", range_.max_commit_ts_);
+  size += 1;
+  return size;
+}
+
+////////////////////////////// FileIndex //////////////////////////////
+
+int FileIndex::insert_index_item(const int64_t file_id,
+    const int64_t min_commit_ts,
+    const int64_t max_commit_ts)
+{
+  int ret = OB_SUCCESS;
+  FileIndexItem *item;
+
+  if (OB_FAIL(alloc_index_item_(item))) {
+    LOG_ERROR("generate new index_item failed", K(item), K(file_id), K(min_commit_ts), K(max_commit_ts));
+  } else if (OB_ISNULL(item)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get null index item", K(item));
+  } else {
+    item->reset(file_id, min_commit_ts, max_commit_ts);
+    if (OB_FAIL(index_array_.push_back(item))) {
+      LOG_ERROR("failed to push back item into index_array", KPC(item));
+    } else {
+      index_file_len_ += item->get_serialize_size();
+    }
+  }
+
+  return ret;
+}
+
+int FileIndex::insert_index_item(const FileIndexItem &item)
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_FAIL(insert_index_item(item.file_id_, item.range_.min_commit_ts_, item.range_.max_commit_ts_))) {
+    LOG_ERROR("failed to insert index_item", K(item));
+  }
+
+  return ret;
+}
+
+int FileIndex::get_index_item(const int64_t file_id, FileIndexItem *&item) const
+{
+  int ret = OB_SUCCESS;
+
+  if (file_id >= index_array_.count() || file_id < 0) {
+    ret = OB_ERROR_OUT_OF_RANGE;
+    LOG_ERROR("file id is out of range", K(file_id), "arr_count", index_array_.count());
+  } else {
+    item = index_array_.at(file_id);
+    if (OB_ISNULL(item)) {
+      LOG_ERROR("get null item from index array", K(item), K(file_id), "arr_count", index_array_.count());
+    } else if (item->file_id_ != file_id) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("the item is not wanted, unexpected", KPC(item), K(file_id));
+    }
+  }
+
+  return ret;
+}
+
+int64_t FileIndex::get_index_file_len() const
+{
+  return index_file_len_;
+}
+
+DEFINE_SERIALIZE(FileIndex)
+{
+  int ret = OB_SUCCESS;
+  ARRAY_FOREACH(index_array_, idx)
+  {
+    FileIndexItem *item = index_array_.at(idx);
+    if (OB_ISNULL(item)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("get null item when calculating serialized size", K(idx), K(item));
+    } else if (OB_FAIL(item->serialize(buf, buf_len, pos))) {
+      LOG_ERROR("failed to serialize item into buf", K(buf_len), K(pos), K(idx));
+    }
+  }
+  return ret;
+}
+
+DEFINE_DESERIALIZE(FileIndex)
+{
+  int ret = OB_SUCCESS;
+  FileIndexItem item;
+  int64_t last_valid_pos = pos;
+  bool done = false;
+  while (OB_SUCC(ret) && !done) {
+    if (OB_FAIL(item.deserialize(buf, data_len, pos))) {
+      if (OB_INVALID_ARGUMENT != ret) {
+        LOG_WARN("failed to deserialize file_index_item", K(data_len), K(pos));
+      }
+      ret = OB_SUCCESS;
+      pos = last_valid_pos;
+      done = true;
+    } else if (OB_FAIL(insert_index_item(item))) {
+      LOG_ERROR("failed to insert item into index array", K(item));
+    } else {
+      last_valid_pos = pos;
+      if (pos >= data_len) {
+        done = true;
+      }
+    }
+  }
+  return ret;
+}
+
+DEFINE_GET_SERIALIZE_SIZE(FileIndex)
+{
+  int64_t size = 0;
+  int ret = OB_SUCCESS;
+  ARRAY_FOREACH(index_array_, idx)
+  {
+    FileIndexItem *item = index_array_.at(idx);
+    if (OB_ISNULL(item)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("get null item when calculating serialized size", K(idx), K(item));
+    } else {
+      size += item->get_serialize_size();
+    }
+  }
+  return size;
+}
+
+
+int FileIndex::alloc_index_item_(FileIndexItem *&item)
+{
+  int ret = OB_SUCCESS;
+
+  void *item_buf = alloc_.alloc(sizeof(FileIndexItem));
+  if (OB_ISNULL(item_buf)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("failed to allocate memory for file index item", K(item_buf));
+  } else {
+    item = new(item_buf) FileIndexItem;
+  }
+
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_file_index.h

@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_FILE_INDEX_H_
+#define OCEANBASE_LOG_MINER_FILE_INDEX_H_
+
+#include "lib/allocator/page_arena.h"
+#include "lib/container/ob_se_array.h"
+#include "lib/ob_define.h"
+#include "lib/utility/ob_print_utils.h"
+#include "ob_log_miner_progress_range.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class FileIndexItem
+{
+public:
+  FileIndexItem() { reset(); }
+  FileIndexItem(const int64_t file_id, const int64_t min_commit_ts, const int64_t max_commit_ts)
+  { reset(file_id, min_commit_ts, max_commit_ts); }
+  ~FileIndexItem() { reset(); }
+
+  void reset() {
+    file_id_ = -1;
+    range_.reset();
+  }
+
+  void reset(const int64_t file_id, const int64_t min_commit_ts, const int64_t max_commit_ts)
+  {
+    file_id_ = file_id;
+    range_.min_commit_ts_ = min_commit_ts;
+    range_.max_commit_ts_ = max_commit_ts;
+  }
+
+  bool operator==(const FileIndexItem &that) const
+  {
+    return file_id_ == that.file_id_ && range_ == that.range_;
+  }
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  TO_STRING_KV(
+    K(file_id_),
+    K(range_)
+  )
+
+  int64_t file_id_;
+  ObLogMinerProgressRange range_;
+};
+
+class FileIndex
+{
+public:
+  FileIndex():
+      alloc_(),
+      index_file_len_(0),
+      index_array_() {}
+  ~FileIndex() { reset(); }
+
+  void reset() {
+    index_file_len_ = 0;
+    index_array_.reset();
+    alloc_.reset();
+  }
+
+  int insert_index_item(const int64_t file_id, const int64_t min_commit_ts, const int64_t max_commit_ts);
+
+  int insert_index_item(const FileIndexItem &item);
+
+  int get_index_item(const int64_t file_id, FileIndexItem *&item) const;
+
+  int64_t get_index_file_len() const;
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  TO_STRING_KV(
+    "file_num", index_array_.count()
+  )
+private:
+  int alloc_index_item_(FileIndexItem *&item);
+
+private:
+  ObArenaAllocator alloc_;
+  int64_t index_file_len_;
+  ObSEArray<FileIndexItem* , 16> index_array_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_file_manager.cpp

@@ -0,0 +1,661 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "share/backup/ob_backup_io_adapter.h"
+#include "ob_log_miner_file_manager.h"
+#include "ob_log_miner_batch_record.h"
+#include "ob_log_miner_args.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+////////////////////////////// ObLogMinerFileManager //////////////////////////////
+
+const char * ObLogMinerFileManager::META_PREFIX = "META/";
+const char * ObLogMinerFileManager::META_EXTENSION = "meta";
+const char * ObLogMinerFileManager::CSV_SUFFIX = "csv";
+const char * ObLogMinerFileManager::SQL_SUFFIX = "sql";
+const char * ObLogMinerFileManager::CONFIG_FNAME = "CONFIG";
+const char * ObLogMinerFileManager::CHECKPOINT_FNAME = "CHECKPOINT";
+const char * ObLogMinerFileManager::INDEX_FNAME = "COMMIT_INDEX";
+const char * ObLogMinerFileManager::EXITCODE_FNAME = "EXITCODE";
+
+ObLogMinerFileManager::ObLogMinerFileManager():
+    is_inited_(false),
+    output_dest_(),
+    mode_(FileMgrMode::INVALID),
+    format_(RecordFileFormat::INVALID),
+    last_write_ckpt_(),
+    meta_map_() { }
+
+ObLogMinerFileManager::~ObLogMinerFileManager()
+{
+  destroy();
+}
+
+int ObLogMinerFileManager::init(const char *path,
+    const RecordFileFormat format,
+    const FileMgrMode mode)
+{
+  int ret = OB_SUCCESS;
+  ObBackupIoAdapter adapter;
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("ObLogMinerFileManager has been initialized", K(is_inited_), K(path), K(format), K(mode));
+  } else if (FileMgrMode::RESTART == mode) {
+    // TODO: restart when interrupted
+    LOG_ERROR("unsupported mode", K(mode));
+  } else if (RecordFileFormat::INVALID == format || FileMgrMode::INVALID == mode) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid record file format or filemgr mode", K(format), K(mode));
+  } else if (OB_FAIL(output_dest_.set(path))) {
+    LOG_ERROR("output dest failed to set path", K(path));
+  } else if (OB_FAIL(meta_map_.init("LogMinerFileIdx"))) {
+    LOG_ERROR("file index init failed");
+  } else if (FileMgrMode::ANALYZE == mode && OB_FAIL(init_path_for_analyzer_())) {
+    LOG_ERROR("init path for analyzer failed", K(path), K(format), K(mode));
+    // TODO: init for flashbacker and restart mode
+  } else {
+    last_write_ckpt_.reset();
+    mode_ = mode;
+    format_ = format;
+    is_inited_ = true;
+    LOG_INFO("ObLogMinerFileManager finished to init", K(path), K(format), K(mode));
+    LOGMINER_STDOUT_V("ObLogMinerFileManager finished to init\n");
+  }
+  return ret;
+}
+
+void ObLogMinerFileManager::destroy()
+{
+  if (IS_INIT) {
+    is_inited_ = false;
+    output_dest_.reset();
+    mode_ = FileMgrMode::INVALID;
+    format_ = RecordFileFormat::INVALID;
+    last_write_ckpt_.reset();
+    meta_map_.destroy();
+    LOG_INFO("ObLogMinerFileManager destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerFileManager destroyed\n");
+  }
+}
+
+int ObLogMinerFileManager::append_records(const ObLogMinerBatchRecord &batch_record)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("file manager hasn't been initialized yet", K(is_inited_));
+  } else {
+    ObLogMinerFileMeta meta;
+    const int64_t file_id = batch_record.get_file_id();
+    const char *data = nullptr;
+    int64_t data_len = 0;
+    batch_record.get_data(data, data_len);
+
+    // make sure to get meta here
+    if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) {
+      if (OB_ENTRY_NOT_EXIST == ret) {
+        if (OB_FAIL(create_data_file_(file_id))) {
+          LOG_ERROR("file manager failed to create file", K(file_id));
+        } else if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) {
+          LOG_ERROR("failed to get file meta after create file", K(file_id));
+        }
+      } else {
+        LOG_ERROR("get meta from meta index failed", K(meta));
+      }
+    }
+
+    // write file in append mode
+    if (OB_SUCC(ret)) {
+      if (0 < data_len && nullptr != data && OB_FAIL(append_data_file_(file_id, data, data_len))) {
+        LOG_ERROR("file manager write file failed", K(batch_record));
+      } else {
+        const ObLogMinerProgressRange &range = batch_record.get_progress_range();
+        meta.data_length_ += data_len;
+        if (OB_INVALID_TIMESTAMP == meta.range_.min_commit_ts_) {
+          meta.range_.min_commit_ts_ = range.min_commit_ts_;
+        }
+
+        if (OB_INVALID_TIMESTAMP == meta.range_.min_commit_ts_ ||
+            range.max_commit_ts_ >= meta.range_.max_commit_ts_) {
+          meta.range_.max_commit_ts_ = range.max_commit_ts_;
+        } else if (range.max_commit_ts_ < meta.range_.max_commit_ts_) {
+          // TODO: just print error, to findout commit version reverse
+          LOG_ERROR("range max_commit_ts rollbacked", K(meta), K(batch_record));
+        }
+
+        if (OB_FAIL(write_meta_(file_id, meta))) {
+          LOG_ERROR("write meta failed when appending batch_record", K(file_id), K(meta));
+        } else if (OB_FAIL(meta_map_.insert_or_update(FileIdWrapper(file_id), meta))) {
+          LOG_ERROR("failed to update meta_map", K(file_id), K(meta));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::write_config(const ObLogMinerArgs &args)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("file manager hasn't been initialized yet", K(is_inited_));
+  } else {
+    char config_file_uri[OB_MAX_URI_LENGTH];
+    const int64_t arg_size = args.get_serialize_size();
+    ObArenaAllocator alloc;
+    char *config_file_data_buf = static_cast<char*>(alloc.alloc(arg_size + 1));
+    int64_t pos = 0;
+    int64_t config_file_uri_len = 0;
+    if (OB_ISNULL(config_file_data_buf)) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_ERROR("allocate memory for config_file_data failed", K(arg_size), K(args));
+    } else if (OB_FAIL(config_file_uri_(config_file_uri, sizeof(config_file_uri), config_file_uri_len))) {
+      LOG_ERROR("failed to get config file uri");
+    } else if (OB_FAIL(args.serialize(config_file_data_buf, arg_size + 1, pos))) {
+      LOG_ERROR("failed to serialize arg into buffer", K(arg_size), K(pos));
+    } else {
+      ObBackupIoAdapter utils;
+      ObString config_file_uri_str(sizeof(config_file_uri), config_file_uri_len, config_file_uri);
+      if OB_FAIL(utils.write_single_file(config_file_uri_str, output_dest_.get_storage_info(),
+          config_file_data_buf, arg_size)) {
+        LOG_ERROR("failed to write config file to dest", K(config_file_uri_str), K(output_dest_), K(arg_size));
+      }
+    }
+
+    if (OB_NOT_NULL(config_file_data_buf)) {
+      alloc.free(config_file_data_buf);
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::write_checkpoint(const ObLogMinerCheckpoint &ckpt)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("file manager hasn't been initialized yet", K(is_inited_));
+  } else {
+    char checkpoint_file_uri[OB_MAX_URI_LENGTH];
+    const int64_t ckpt_size = ckpt.get_serialize_size();
+    ObArenaAllocator alloc;
+    char *ckpt_buf = static_cast<char*>(alloc.alloc(ckpt_size + 1));
+    int64_t pos = 0;
+    int64_t checkpoint_file_uri_len = 0;
+    if (OB_ISNULL(ckpt_buf)) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_ERROR("allocte memory for checkpoint failed", K(ckpt_size), K(ckpt));
+    } else if (OB_FAIL(checkpoint_file_uri_(checkpoint_file_uri,
+        sizeof(checkpoint_file_uri), checkpoint_file_uri_len))) {
+      LOG_ERROR("get checkpoint file uri failed", K(checkpoint_file_uri_len));
+    } else if (OB_FAIL(ckpt.serialize(ckpt_buf, ckpt_size+1, pos))) {
+      LOG_ERROR("ckpt failed to serialize to ckpt_buf", K(ckpt), K(ckpt_size), K(pos));
+    } else {
+      // TODO: remove meta from meta_index according to max_file_id when writing checkpoint
+      ObBackupIoAdapter utils;
+      ObString ckpt_file_uri_str(sizeof(checkpoint_file_uri), checkpoint_file_uri_len, checkpoint_file_uri);
+      if (OB_FAIL(utils.write_single_file(ckpt_file_uri_str, output_dest_.get_storage_info(),
+          ckpt_buf, ckpt_size))) {
+        LOG_ERROR("failed to write config file to dest", K(checkpoint_file_uri), K(output_dest_), K(ckpt_size));
+      } else if (OB_FAIL(update_last_write_ckpt_(ckpt))) {
+        LOG_ERROR("update last write ckpt failed", K(ckpt), K(last_write_ckpt_));
+      }
+    }
+
+    if (OB_NOT_NULL(ckpt_buf)) {
+      alloc.free(ckpt_buf);
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerFileManager::read_file(const int64_t file_id, char *buf, const int64_t buf_len)
+{
+  int ret = OB_SUCCESS;
+  // TODO : read single file
+  return ret;
+}
+
+int ObLogMinerFileManager::get_file_range(const int64_t min_timestamp_us,
+    const int64_t max_timestamp_us,
+    int64_t &min_file_id,
+    int64_t &max_file_id)
+{
+  int ret = OB_SUCCESS;
+  // TODO: flashback mode, get file range according to min_timestamp_us and max_timestamp_us
+  return ret;
+}
+
+int ObLogMinerFileManager::read_config(ObLogMinerArgs &args)
+{
+  int ret = OB_SUCCESS;
+  // TODO: restart mode, read config to recover from exception
+  return ret;
+}
+
+int ObLogMinerFileManager::read_checkpoint(ObLogMinerCheckpoint &ckpt)
+{
+  int ret = OB_SUCCESS;
+  // TODO: flashback/restart mode, get checkpoint
+  return ret;
+}
+
+int ObLogMinerFileManager::config_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s",
+      output_dest_.get_root_path().ptr(), CONFIG_FNAME))) {
+    LOG_ERROR("fill config file uri into buffer failed", K(output_dest_), K(buf_len));
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::checkpoint_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s",
+      output_dest_.get_root_path().ptr(), CHECKPOINT_FNAME))) {
+    LOG_ERROR("fill checkpoint file uri into buffer failed", K(output_dest_), K(buf_len));
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::data_file_uri_(
+    const int64_t file_id,
+    char *buf,
+    const int64_t buf_len,
+    int64_t &length) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%ld.%s",
+      output_dest_.get_root_path().ptr(), file_id, data_file_extension_()))) {
+    LOG_ERROR("failed to get data file uri", K(length), K(file_id), K(buf_len));
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::meta_file_uri_(
+    const int64_t file_id,
+    char *buf,
+    const int64_t buf_len,
+    int64_t &length) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s%ld.%s",
+      output_dest_.get_root_path().ptr(), META_PREFIX, file_id, META_EXTENSION))) {
+    LOG_ERROR("failed to get meta file uri", K(length), K(file_id), K(buf_len));
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::index_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s",
+      output_dest_.get_root_path().ptr(), INDEX_FNAME))) {
+    LOG_ERROR("fill index file uri into buffer failed", K(output_dest_), K(buf_len));
+  }
+  return ret;
+}
+
+const char *ObLogMinerFileManager::data_file_extension_() const
+{
+  const char *suffix = "";
+  switch(format_) {
+    case RecordFileFormat::CSV: {
+      suffix = "csv";
+      break;
+    }
+
+    case RecordFileFormat::REDO_ONLY:
+    case RecordFileFormat::UNDO_ONLY: {
+      suffix = "sql";
+      break;
+    }
+
+    case RecordFileFormat::JSON: {
+      suffix = "json";
+      break;
+    }
+
+    case RecordFileFormat::AVRO: {
+      suffix = "avro";
+      break;
+    }
+
+    case RecordFileFormat::PARQUET: {
+      suffix = "parquet";
+      break;
+    }
+
+    default: {
+      suffix = "";
+      break;
+    }
+  }
+  return suffix;
+}
+
+int ObLogMinerFileManager::create_data_file_(const int64_t file_id)
+{
+  int ret = OB_SUCCESS;
+  // create file, generate file header first.
+  int64_t pos = 0;
+  ObLogMinerFileMeta meta;
+  ObArenaAllocator alloc;
+  char *header_data = nullptr;
+  int64_t header_len = 0;
+  char data_file_uri[OB_MAX_URI_LENGTH];
+  int64_t uri_len = 0;
+
+  // create file meta, no need to write metainfo into storage
+  // but it's okay to write meta here
+  if (OB_FAIL(generate_data_file_header_(alloc, header_data, header_len))) {
+    LOG_ERROR("failed to generate file header", K(format_), K(header_len));
+  } else if (OB_FAIL(data_file_uri_(file_id, data_file_uri, sizeof(data_file_uri), uri_len))) {
+    LOG_ERROR("failed to get data_file uri", K(file_id), K(uri_len));
+  } else if (OB_FAIL(append_file_(data_file_uri, 0, header_data, header_len))) {
+    LOG_ERROR("failed to append file when creating file", K(data_file_uri), K(file_id), K(header_len));
+  } else {
+    meta.range_.min_commit_ts_ = OB_INVALID_TIMESTAMP;
+    meta.range_.max_commit_ts_ = OB_INVALID_TIMESTAMP;
+    meta.data_length_ = header_len;
+    if (OB_FAIL(write_meta_(file_id, meta))) {
+      LOG_ERROR("failed to write meta when writing file", K(file_id), K(meta));
+    } else if (OB_FAIL(meta_map_.insert(file_id, meta))) {
+      LOG_ERROR("failed to insert meta into meta_index", K(file_id), K(meta));
+    }
+  }
+
+  if (OB_NOT_NULL(header_data)) {
+    alloc.free(header_data);
+  }
+
+  return ret;
+}
+
+int ObLogMinerFileManager::generate_data_file_header_(
+    ObIAllocator &alloc,
+    char *&data,
+    int64_t &data_len)
+{
+  int ret = OB_SUCCESS;
+  switch (format_) {
+    case RecordFileFormat::CSV: {
+      #define MINER_SCHEMA_DEF(field, id, args...) \
+        #field,
+      const char *header[] = {
+        #include "ob_log_miner_analyze_schema.h"
+      };
+      #undef MINER_SCHEMA_DEF
+      int64_t header_len = 0, element_cnt = sizeof(header)/sizeof(const char*);
+      char *buf = nullptr;
+      int64_t pos = 0;
+      for (int i = 0; i < element_cnt; i++) {
+        header_len += strlen(header[i]) + 1;
+      }
+      if (OB_ISNULL(buf = static_cast<char*>(alloc.alloc(header_len + 1)))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_ERROR("allocate memory for csv header failed", K(header_len));
+      } else {
+        for (int i = 0; OB_SUCC(ret) && i < element_cnt; i++) {
+          char delimiter = i == element_cnt-1 ? '\n': ',';
+          if (OB_FAIL(databuff_printf(buf, header_len + 1, pos, "%s%c", header[i], delimiter))) {
+            LOG_ERROR("failed to fill header into buffer", K(i), K(header[i]), K(pos));
+          }
+        }
+
+        if (OB_SUCC(ret)) {
+          data = buf;
+          data_len = header_len;
+        }
+      }
+      break;
+    }
+
+    case RecordFileFormat::REDO_ONLY:
+    case RecordFileFormat::UNDO_ONLY:
+    case RecordFileFormat::JSON: {
+      // do nothing;
+      break;
+    }
+
+    // TODO: support other type
+    default: {
+      ret = OB_NOT_SUPPORTED;
+      break;
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::append_data_file_(const int64_t file_id, const char *data, const int64_t data_len)
+{
+  int ret = OB_SUCCESS;
+
+  ObBackupIoAdapter utils;
+  char data_file_uri[OB_MAX_URI_LENGTH];
+  int64_t uri_len = 0;
+  ObIOFd fd;
+  ObIODevice *device_handle = nullptr;
+  ObLogMinerFileMeta meta;
+  int64_t write_size = 0;
+
+  if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) {
+    LOG_ERROR("failed to get meta in meta_index when trying to append data to file", K(file_id),
+        K(data_len), K(meta));
+  } else if (OB_FAIL(data_file_uri_(file_id, data_file_uri, sizeof(data_file_uri), uri_len))) {
+    LOG_ERROR("failed to get data_file uri", K(file_id), K(uri_len), K(output_dest_));
+  } else if (OB_FAIL(append_file_(data_file_uri, meta.data_length_, data, data_len))) {
+    LOG_ERROR("failed to open device", K(file_id), K(output_dest_), K(data_file_uri));
+  }
+
+  return ret;
+}
+
+int ObLogMinerFileManager::append_file_(const ObString &uri,
+    const int64_t offset,
+    const char *data,
+    const int64_t data_len)
+{
+  int ret = OB_SUCCESS;
+
+  ObBackupIoAdapter utils;
+  ObIOFd fd;
+  ObIODevice *device_handle = nullptr;
+  int64_t write_size = 0;
+  if (nullptr == data || 0 == data_len) {
+    // do nothing
+  } else if (OB_FAIL(utils.open_with_access_type(device_handle, fd, output_dest_.get_storage_info(),
+      uri, common::OB_STORAGE_ACCESS_RANDOMWRITER))) {
+    LOG_ERROR("failed to open device", K(uri), K(output_dest_), K(uri));
+  } else if (OB_FAIL(device_handle->pwrite(fd, offset, data_len, data, write_size))) {
+    LOG_ERROR("failed to write data into file", K(uri), K(output_dest_),
+        K(data_len), K(write_size));
+  } else if (write_size != data_len) {
+    ret = OB_IO_ERROR;
+    LOG_WARN("write length not equal to data length", K(write_size), K(data_len));
+  }
+
+  if (nullptr != device_handle && fd.is_valid()) {
+    int tmp_ret = OB_SUCCESS;
+    if (OB_TMP_FAIL(utils.close_device_and_fd(device_handle, fd))) {
+      LOG_ERROR("failed to close device and fd", K(uri), K(fd));
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerFileManager::read_data_file_(const int64_t file_id, char *data, const int64_t data_len)
+{
+  int ret = OB_SUCCESS;
+  // TODO: read the whole file into the buf(data)
+  return ret;
+}
+
+int ObLogMinerFileManager::write_data_file_(const int64_t file_id, const char *data, const int64_t data_len)
+{
+  int ret = OB_SUCCESS;
+  // TODO: write the whole file with data, used in restart mode.
+  return ret;
+}
+
+int ObLogMinerFileManager::read_meta_(const int64_t file_id, ObLogMinerFileMeta &meta)
+{
+  int ret = OB_SUCCESS;
+  // TODO: read meta to read file.
+  return ret;
+}
+
+int ObLogMinerFileManager::write_meta_(const int64_t file_id, const ObLogMinerFileMeta &meta)
+{
+  int ret = OB_SUCCESS;
+  char meta_uri[OB_MAX_URI_LENGTH];
+  ObArenaAllocator alloc;
+  int64_t uri_len = 0;
+  ObBackupIoAdapter utils;
+  const int64_t buf_len = meta.get_serialize_size();
+  int64_t pos = 0;
+  char *buf = static_cast<char*>(alloc.alloc(buf_len + 1));
+
+  if (OB_ISNULL(buf)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("failed to allocate memory to file meta data buffer", K(buf_len), K(buf));
+  } else if (OB_FAIL(meta.serialize(buf, buf_len + 1, pos))) {
+    LOG_ERROR("failed to serialize meta into buffer", K(buf_len), K(meta));
+  } else if (OB_FAIL(meta_file_uri_(file_id, meta_uri, sizeof(meta_uri), uri_len))) {
+    LOG_ERROR("failed to get meta_file uri", K(file_id), K(uri_len));
+  } else if (OB_FAIL(utils.write_single_file(meta_uri,
+      output_dest_.get_storage_info(), buf, buf_len))) {
+    LOG_ERROR("failed to write meta file", K(meta), K(file_id), K(meta_uri), K(buf_len));
+  }
+
+  if (OB_NOT_NULL(buf)) {
+    alloc.free(buf);
+  }
+
+  return ret;
+}
+
+int ObLogMinerFileManager::write_index_(const int64_t file_id)
+{
+  int ret = OB_SUCCESS;
+
+  ObLogMinerFileMeta meta;
+  char index_uri_cstr[OB_MAX_URI_LENGTH];
+  int64_t index_uri_len = 0;
+
+  if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) {
+    LOG_ERROR("get meta from meta_index failed", K(file_id), K(meta));
+  } else if (OB_FAIL(index_file_uri_(index_uri_cstr, sizeof(index_uri_cstr), index_uri_len))) {
+    LOG_ERROR("failed to get index file uri", K(output_dest_), K(file_id), K(index_uri_len));
+  } else {
+    const int64_t index_file_offset = file_index_.get_index_file_len();
+    FileIndexItem item(file_id, meta.range_.min_commit_ts_, meta.range_.max_commit_ts_);
+    const int64_t index_data_len = item.get_serialize_size();
+    ObArenaAllocator alloc;
+    int64_t pos = 0;
+    char *item_buf = static_cast<char*>(alloc.alloc(index_data_len + 1));
+    if (OB_ISNULL(item_buf)) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_ERROR("allocate memory for fileIndexItem failed", K(item_buf), K(index_data_len), K(item));
+    } else if (OB_FAIL(item.serialize(item_buf, index_data_len + 1, pos))) {
+      LOG_ERROR("failed to serialize index item into item_buf", K(index_data_len), K(pos));
+    } else if (OB_FAIL(append_file_(index_uri_cstr, index_file_offset, item_buf, index_data_len))) {
+      LOG_ERROR("failed to append index item data into storage system", K(index_uri_cstr), K(index_file_offset),
+          K(index_data_len));
+    } else if (OB_FAIL(file_index_.insert_index_item(item))) {
+      LOG_ERROR("failed to insert index itme into file_index", K(file_index_), K(item));
+    }
+
+  }
+
+  return ret;
+}
+
+int ObLogMinerFileManager::update_last_write_ckpt_(const ObLogMinerCheckpoint &ckpt)
+{
+  int ret = OB_SUCCESS;
+
+  const int64_t last_max_file_id = last_write_ckpt_.max_file_id_;
+  const int64_t cur_max_file_id = ckpt.max_file_id_;
+
+  // traverse from last_max_file_id+1 to cur_max_file_id
+  for (int64_t i = last_max_file_id + 1; OB_SUCC(ret) && i <= cur_max_file_id; i++) {
+    if (OB_FAIL(write_index_(i))) {
+      LOG_ERROR("failed to write index for data file", "file_id", i, K(last_max_file_id), K(cur_max_file_id));
+    } else if (OB_FAIL(meta_map_.erase(i))) {
+      LOG_ERROR("failed to erase meta from meta_index", "file_id", i, K(last_max_file_id), K(cur_max_file_id));
+    } else {
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    last_write_ckpt_ = ckpt;
+  }
+
+  return ret;
+}
+
+int ObLogMinerFileManager::init_path_for_analyzer_()
+{
+  int ret = OB_SUCCESS;
+
+  ObBackupIoAdapter adapter;
+  char path_uri[OB_MAX_URI_LENGTH];
+  bool is_exist = false;
+
+  if (OB_FAIL(adapter.is_exist(output_dest_.get_root_path(),
+      output_dest_.get_storage_info(), is_exist))) {
+    LOG_ERROR("failed to check output_dest existence", K(output_dest_));
+  } else if (! is_exist) {
+    if (OB_FAIL(adapter.mkdir(output_dest_.get_root_path(), output_dest_.get_storage_info()))) {
+      LOG_ERROR("failed to make output_dest", K(output_dest_));
+    }
+  }
+  // oss don't create directory actually, so is_empty_directory() must be checked.
+  if(OB_SUCC(ret)) {
+    bool is_empty = false;
+    if (OB_FAIL(adapter.is_empty_directory(output_dest_.get_root_path(),
+        output_dest_.get_storage_info(), is_empty))) {
+      LOG_ERROR("failed to check is empty directory", K(output_dest_));
+    } else if (!is_empty) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("init for analyzer but not empty directory", K(output_dest_), K(is_empty));
+      LOGMINER_STDOUT("not empty output directory: %s\n", output_dest_.get_root_path().ptr());
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(databuff_printf(path_uri, sizeof(path_uri), "%s/%s",
+        output_dest_.get_root_path().ptr(), META_PREFIX))) {
+      LOG_ERROR("failed to fill path for meta entry", K(output_dest_), K(META_PREFIX));
+    } else if (OB_FAIL(adapter.mkdir(path_uri, output_dest_.get_storage_info()))) {
+      LOG_ERROR("failed to mkdir for meta", K(path_uri), K(output_dest_));
+    }
+  }
+
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_file_manager.h

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_FILE_MANAGER_H_
+#define OCEANBASE_LOG_MINER_FILE_MANAGER_H_
+
+#include "lib/hash/ob_linear_hash_map.h"
+#include "share/backup/ob_backup_struct.h"
+#include "ob_log_miner_file_index.h"
+#include "ob_log_miner_progress_range.h"
+#include "ob_log_miner_file_meta.h"
+#include "ob_log_miner_analyzer_checkpoint.h"
+#include "ob_log_miner_record_file_format.h"
+
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerArgs;
+class ObLogMinerBatchRecord;
+
+class ILogMinerFileManager
+{
+public:
+  // write operation must satisfy the following order
+  // 1. write_config when logminer starts
+  // 2. write_checkpoint after append_record
+  // expect no parallel append to one file
+  virtual int append_records(const ObLogMinerBatchRecord &batch_record) = 0;
+  virtual int write_config(const ObLogMinerArgs &args) = 0;
+  virtual int write_checkpoint(const ObLogMinerCheckpoint &ckpt) = 0;
+  // read operation must satisfy the following order
+  // 1. read_config when logminer restarts
+  // 2. read_checkpoint before get_file_range to make sure get valid file range
+  // 3. read_checkpoint before read_file to make sure read valid file
+  virtual int read_file(const int64_t file_id, char *buf, const int64_t buf_len) = 0;
+  virtual int get_file_range(const int64_t min_timestamp_us,
+              const int64_t max_timestamp_us,
+              int64_t &min_file_id,
+              int64_t &max_file_id) = 0;
+  virtual int read_config(ObLogMinerArgs &args) = 0;
+  // read_checkpoint should be used only when logminer restarts
+  // there should not be concurrency issue with write_checkpoint()
+  virtual int read_checkpoint(ObLogMinerCheckpoint &ckpt) = 0;
+};
+
+struct FileIdWrapper {
+  FileIdWrapper(const int64_t file_id): fid_(file_id) {}
+
+  int hash(uint64_t &val) const {
+    val = fid_;
+    return OB_SUCCESS;
+  }
+
+  bool operator==(const FileIdWrapper &that) const {
+    return fid_ == that.fid_;
+  }
+
+  int64_t fid_;
+};
+
+typedef ObLinearHashMap<FileIdWrapper, ObLogMinerFileMeta>  FileMetaMap;
+
+class ObLogMinerFileManager: public ILogMinerFileManager
+{
+public:
+  enum class FileMgrMode {
+    INVALID = -1,
+    ANALYZE = 0,
+    FLASHBACK,
+    RESTART
+  };
+  static const char *META_PREFIX;
+  static const char *META_EXTENSION;
+  static const char *CSV_SUFFIX;
+  static const char *SQL_SUFFIX;
+  static const char *CONFIG_FNAME;
+  static const char *CHECKPOINT_FNAME;
+  static const char *INDEX_FNAME;
+  static const char *EXITCODE_FNAME;
+public:
+  virtual int append_records(const ObLogMinerBatchRecord &batch_record);
+  virtual int write_config(const ObLogMinerArgs &args);
+  virtual int write_checkpoint(const ObLogMinerCheckpoint &ckpt);
+  virtual int read_file(const int64_t file_id, char *buf, const int64_t buf_len);
+  virtual int get_file_range(const int64_t min_timestamp_us,
+              const int64_t max_timestamp_us,
+              int64_t &min_file_id,
+              int64_t &max_file_id);
+  virtual int read_config(ObLogMinerArgs &args);
+  virtual int read_checkpoint(ObLogMinerCheckpoint &ckpt);
+
+public:
+  ObLogMinerFileManager();
+  virtual ~ObLogMinerFileManager();
+  int init(const char *path,
+           const RecordFileFormat format,
+           const FileMgrMode mode);
+  void destroy();
+
+private:
+  int config_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const;
+  int checkpoint_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const;
+  int data_file_uri_(const int64_t file_id, char *buf, const int64_t buf_len, int64_t &length) const;
+  int meta_file_uri_(const int64_t file_id, char *buf, const int64_t buf_len, int64_t &length) const;
+  int index_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const;
+  const char *data_file_extension_() const;
+
+  int create_data_file_(const int64_t file_id);
+  int generate_data_file_header_(ObIAllocator &alloc, char *&data, int64_t &data_len);
+  int append_data_file_(const int64_t file_id, const char *data, const int64_t data_len);
+  int append_file_(const ObString &uri, const int64_t offset, const char *data, const int64_t data_len);
+  int read_data_file_(const int64_t file_id, char *data, const int64_t data_len);
+  // overwrite the whole file with data
+  int write_data_file_(const int64_t file_id, const char *data, const int64_t data_len);
+  int read_meta_(const int64_t file_id, ObLogMinerFileMeta &meta);
+  int write_meta_(const int64_t file_id, const ObLogMinerFileMeta &meta);
+  // int file_exists_(const char *file_path) const;
+
+  // only called by write_checkpoint
+  int write_index_(const int64_t file_id);
+  // only called by write_checkpoint
+  int update_last_write_ckpt_(const ObLogMinerCheckpoint &ckpt);
+
+  int init_path_for_analyzer_();
+
+private:
+  bool is_inited_;
+  share::ObBackupDest output_dest_;
+  FileMgrMode mode_;
+  RecordFileFormat format_;
+  ObLogMinerCheckpoint last_write_ckpt_;
+  FileMetaMap meta_map_;
+  FileIndex file_index_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_file_meta.cpp

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_file_meta.h"
+#include "ob_log_miner_utils.h"
+////////////////////////////// ObLogMinerFileMeta //////////////////////////////
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+const char *ObLogMinerFileMeta::data_len_key = "DATA_LEN";
+
+int ObLogMinerFileMeta::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(range_.serialize(buf, buf_len, pos))) {
+    LOG_ERROR("failed to serialize range into buf", K(range_), K(buf_len), K(pos));
+  } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n",
+      data_len_key, data_length_))) {
+    LOG_ERROR("failed to fill data_len into buf", K(buf_len), K(pos), K(data_length_));
+  }
+  return ret;
+}
+
+int ObLogMinerFileMeta::deserialize(const char *buf, const int64_t data_len, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(range_.deserialize(buf, data_len, pos))) {
+    LOG_ERROR("failed to deserialize range", K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(data_len_key, buf, data_len, pos, data_length_))) {
+    LOG_ERROR("failed to deserialize data_length", K(data_len_key), K(data_len), K(pos), K(data_length_));
+  }
+
+  return ret;
+}
+
+int64_t ObLogMinerFileMeta::get_serialize_size() const
+{
+  int64_t size = 0;
+  const int64_t digit_max_len = 30;
+  char digit_str[digit_max_len];
+  size += range_.get_serialize_size();
+  size += strlen(data_len_key) + 2;
+  size += snprintf(digit_str, digit_max_len, "%ld", data_length_);
+  return size;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_file_meta.h

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_FILE_META_H_
+#define OCEANBASE_LOG_MINER_FILE_META_H_
+
+#include "lib/ob_define.h"
+#include "lib/utility/ob_print_utils.h"
+#include "ob_log_miner_progress_range.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+struct ObLogMinerFileMeta
+{
+public:
+
+  static const char *data_len_key;
+
+  ObLogMinerFileMeta() {reset();}
+  ~ObLogMinerFileMeta() {reset();}
+
+  void reset() {
+    range_.reset();
+    data_length_ = 0;
+  }
+
+  bool operator==(const ObLogMinerFileMeta &that) const
+  {
+    return range_ == that.range_ && data_length_ == that.data_length_;
+  }
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  TO_STRING_KV(
+    K(range_),
+    K(data_length_)
+  )
+
+public:
+  ObLogMinerProgressRange range_;
+  int64_t data_length_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_filter_condition.cpp

@@ -0,0 +1,427 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ *
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_filter_condition.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+int ObLogMinerColVal::init(const ObString &column_name,
+    const ObString &value,
+    const bool is_null,
+    ObIAllocator *alloc)
+{
+  int ret = OB_SUCCESS;
+  char *value_buf = nullptr;
+  allocator_ = alloc;
+  if (OB_ISNULL(alloc)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get null allocator, unexpected", K(alloc), K(column_name), K(value));
+  } else if (OB_FAIL(col_.assign(column_name))) {
+    LOG_ERROR("assign column_name failed", K(column_name));
+  } else {
+    if (is_null) {
+      is_null_ = true;
+    } else {
+      if (OB_ISNULL(value_buf = static_cast<char *>(alloc->alloc(value.length() + 1)))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_ERROR("allocate memory failed for value buf", K(value));
+      } else {
+        MEMCPY(value_buf, value.ptr(), value.length());
+        value_buf[value.length()] = '\0';
+        val_.assign(value_buf, value.length());
+        is_null_ = false;
+      }
+    }
+  }
+  return ret;
+}
+
+void ObLogMinerColVal::destroy()
+{
+  col_.reset();
+  if (nullptr != allocator_) {
+    allocator_->free(val_.ptr());
+  }
+  allocator_ = nullptr;
+}
+
+int DbAndTableWrapper::hash(uint64_t &val) const
+{
+    uint64_t hash_val = 0;
+    hash_val = common::murmurhash(db_name_.ptr(), db_name_.size(), hash_val);
+    hash_val = common::murmurhash(table_name_.ptr(), table_name_.size(), hash_val);
+    return OB_SUCCESS;
+  }
+
+const char *ObLogMinerTableColumnCond::DATABASE_KEY = "database_name";
+const char *ObLogMinerTableColumnCond::TABLE_KEY = "table_name";
+const char *ObLogMinerTableColumnCond::COLUMN_COND_KEY = "column_cond";
+
+int ObLogMinerTableColumnCond::init(ObIAllocator *alloc, json::Object &obj)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(alloc)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("allocator fo oblogminer table cond is null", K(alloc));
+  } else {
+    alloc_ = alloc;
+  }
+  DLIST_FOREACH(it, obj) {
+    if (OB_ISNULL(it)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_ERROR("get nulll pair in json, unexpected", K(it));
+    } else if (0 == it->name_.case_compare(DATABASE_KEY)) {
+      if (OB_ISNULL(it->value_)) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get database key get null value", "key", it->name_,
+            "value", it->value_);
+      } else if (json::JT_STRING != it->value_->get_type()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get database key but value type is not expected", "key", it->name_,
+            "value", it->value_, "value_type", it->value_->get_type());
+      } else if (! db_name_.is_empty()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get database key but db_name has been set already", K(db_name_), "key", it->name_,
+            "value", it->value_->get_string());
+      } else if (OB_FAIL(db_name_.assign(it->value_->get_string()))) {
+        LOG_ERROR("assign db_name failed", K(db_name_), "value", it->value_->get_string());
+      } else {
+        // succ
+      }
+    } else if (0 == it->name_.case_compare(TABLE_KEY)) {
+      if (OB_ISNULL(it->value_)) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get table key get null value", "key", it->name_,
+            "value", it->value_);
+      } else if (json::JT_STRING != it->value_->get_type()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get table key but value type is not expected", "key", it->name_,
+            "value", it->value_, "value_type", it->value_->get_type());
+      } else if (! table_name_.is_empty()) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get table key but table_name has been set already", K(table_name_), "key", it->name_,
+            "value", it->value_->get_string());
+      } else if (OB_FAIL(table_name_.assign(it->value_->get_string()))) {
+        LOG_ERROR("assign table_name failed", K(table_name_), "value", it->value_->get_string());
+      } else {
+        // succ
+      }
+    } else if (0 == it->name_.case_compare(COLUMN_COND_KEY)) {
+      if (OB_NOT_NULL(it->value_)) {
+        if (json::JT_ARRAY != it->value_->get_type()) {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_ERROR("get condtion key, expect value of array type, but got unexpected type", "key", it->name_,
+              "value_type", it->value_->get_type());
+        } else {
+          DLIST_FOREACH(col_cond, it->value_->get_array()) {
+            if (OB_ISNULL(col_cond)) {
+              ret = OB_ERR_UNEXPECTED;
+              LOG_ERROR("column_cond key is specified, get null column cond, unexpetced", "key", it->name_,
+                  K(col_cond));
+            } else if (json::JT_OBJECT != col_cond->get_type()) {
+              ret = OB_INVALID_ARGUMENT;
+              LOG_ERROR("get column cond, but its type is not as expected, object type is expected",
+                  "column_cond_type", col_cond->get_type());
+            } else if (OB_FAIL(add_column_cond_(col_cond->get_object()))) {
+              LOG_ERROR("add column condition faile", "col_cond", col_cond->get_object());
+            } else {
+              // succ
+            }
+          }
+        }
+      }
+    } else {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("get invalid key for table cond", "key", it->name_);
+    }
+  }
+
+  if (table_name_.is_empty() || db_name_.is_empty() || column_conds_.empty()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("It's unexpected to not specify column_conds or table_name or db_name", K(table_name_),
+        K(db_name_), K(column_conds_));
+  }
+  return ret;
+}
+
+int ObLogMinerTableColumnCond::init(ObIAllocator *alloc,
+    const ObString &db,
+    const ObString &tbl)
+{
+  int ret = OB_SUCCESS;
+
+  alloc_ = alloc;
+  if (OB_FAIL(db_name_.assign(db))) {
+    LOG_ERROR("failed to assign db into db_name", K(db));
+  } else if (OB_FAIL(table_name_.assign(tbl))) {
+    LOG_ERROR("failed to assign tbl into table_name", K(tbl));
+  }
+
+  return ret;
+}
+
+int ObLogMinerTableColumnCond::add_column_cond_(json::Object &obj)
+{
+  int ret = OB_SUCCESS;
+  ObLogMinerColumnVals col_cond;
+  DLIST_FOREACH(col_val, obj) {
+    ObLogMinerColVal col_val_pair;
+    bool is_null_value = json::JT_NULL == col_val->value_->get_type();
+    if (OB_ISNULL(col_val)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("get null col_val, unexpected for json parser", K(col_val));
+    } else if (OB_ISNULL(col_val->value_)) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("get invalid column condition, whose value for column is null", "column_name", col_val->name_);
+    } else if (json::JT_STRING != col_val->value_->get_type() && !is_null_value) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("get unexpected column value type", "column_name", col_val->name_,
+          "column_val_type", col_val->value_->get_type());
+    } else if (OB_FAIL(col_val_pair.init(col_val->name_,
+        col_val->value_->get_string(), is_null_value, alloc_))) {
+      LOG_ERROR("column_name-column_value pair failed to init", "name", col_val->name_,
+          "value", col_val->value_->get_string());
+    } else if (OB_FAIL(col_cond.push_back(col_val_pair))) {
+      LOG_ERROR("failed to push back col_val_pair into column_conds", K(col_val_pair));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(column_conds_.push_back(col_cond))) {
+      LOG_ERROR("failed to push back column_cond into column cond array", K(col_cond));
+    }
+  }
+
+  return ret;
+}
+
+void ObLogMinerTableColumnCond::destroy()
+{
+  db_name_.reset();
+  table_name_.reset();
+  column_conds_.destroy();
+  alloc_ = nullptr;
+}
+
+uint64_t ObLogMinerTableColumnCond::hash() const
+{
+  uint64_t hash_val = 0;
+  hash_val = common::murmurhash(db_name_.ptr(), db_name_.size(), hash_val);
+  hash_val = common::murmurhash(table_name_.ptr(), table_name_.size(), hash_val);
+  return hash_val;
+}
+
+int ObLogMinerTableColumnCond::hash(uint64_t &hash_val) const
+{
+  hash_val = hash();
+  return OB_SUCCESS;
+}
+
+int ObLogMinerMultiTableColumnCond::add_table_cond_(json::Object &obj)
+{
+  int ret = OB_SUCCESS;
+  ObLogMinerTableColumnCond *cond =
+      static_cast<ObLogMinerTableColumnCond*>(alloc_->alloc(sizeof(ObLogMinerTableColumnCond)));
+  if (OB_ISNULL(cond)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("allocate memory for ObLogMinerTableColumnCond failed", K(cond));
+  } else {
+    cond = new (cond) ObLogMinerTableColumnCond;
+    if (OB_FAIL(cond->init(alloc_, obj))) {
+      LOG_ERROR("table_cond init failed", KP(cond));
+    } else if (OB_FAIL(table_column_conds_.insert(
+        DbAndTableWrapper(cond->db_name_.ptr(), cond->table_name_.ptr()), cond))) {
+      LOG_ERROR("failed to push back table cond into table_cond_array", K(cond));
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerMultiTableColumnCond::parse_json_cond_(const char *table_cond_str)
+{
+  int ret = OB_SUCCESS;
+  ObArenaAllocator allocator(ObModIds::JSON_PARSER);
+  json::Parser parser;
+  json::Value *root = nullptr;
+  if (OB_ISNULL(table_cond_str)) {
+    LOG_INFO("get empty table cond str, don't filter any table", K(table_cond_str));
+  } else if (OB_FAIL(parser.init(&allocator))) {
+    LOG_ERROR("init json parser for multi table cond failed");
+  } else if (OB_FAIL(parser.parse(table_cond_str, strlen(table_cond_str), root))) {
+    LOG_ERROR("parse table_cond failed", K(table_cond_str), K(root));
+    // rewrite the error code
+    ret = OB_INVALID_ARGUMENT;
+  } else {
+    if (json::JT_ARRAY == root->get_type()) {
+      DLIST_FOREACH(it, root->get_array()) {
+        if (OB_ISNULL(it)) {
+          ret = OB_ERR_UNEXPECTED;
+        } else if (json::JT_OBJECT == it->get_type()) {
+          if (OB_FAIL(add_table_cond_(it->get_object()))) {
+            LOG_ERROR("failed to add table cond", K(table_cond_str));
+          } else {
+            // succ
+          }
+        } else {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_ERROR("get unexpected table cond type in json", "type", it->get_type());
+        }
+      }
+    } else {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_ERROR("get unexpected root type in json, expected array", "type", root->get_type());
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerMultiTableColumnCond::init(const char *table_cond_str)
+{
+  int ret = OB_SUCCESS;
+  ObArenaAllocator allocator(ObModIds::JSON_PARSER);
+  json::Parser parser;
+  json::Value *root = nullptr;
+  if (OB_ISNULL(table_cond_str) || 0 == strcmp(table_cond_str, "")) {
+    LOG_INFO("get empty table cond str, don't filter any table", K(table_cond_str));
+  } else if (OB_FAIL(table_column_conds_.init("LogMnrColCond"))) {
+    LOG_ERROR("failed to init table_column_conds_", K(ret));
+  } else if (OB_FAIL(parse_json_cond_(table_cond_str))) {
+    LOG_ERROR("failed to parse json cond", K(table_cond_str));
+  }
+
+  return ret;
+}
+
+void ObLogMinerMultiTableColumnCond::destroy()
+{
+  table_column_conds_.destroy();
+}
+
+const char *ObLogMinerOpCond::INSERT_OP_STR = "insert";
+const char *ObLogMinerOpCond::UPDATE_OP_STR = "update";
+const char *ObLogMinerOpCond::DELETE_OP_STR = "delete";
+const char *ObLogMinerOpCond::OP_DELIMITER = "|";
+
+int64_t ObLogMinerOpCond::record_type_to_bitcode(const RecordType type) {
+  int64_t ret = UNKNOWN_BITCODE;
+  switch (type) {
+    case EINSERT: {
+      ret = INSERT_BITCODE;
+      break;
+    }
+
+    case EUPDATE: {
+      ret = UPDATE_BITCODE;
+      break;
+    }
+
+    case EDELETE: {
+      ret = DELETE_BITCODE;
+      break;
+    }
+
+    case EDDL: {
+      ret = DDL_BITCODE;
+      break;
+    }
+
+    case HEARTBEAT: {
+      ret = HEARTBEAT_BITCODE;
+      break;
+    }
+
+    case EBEGIN: {
+      ret = BEGIN_BITCODE;
+      break;
+    }
+
+    case ECOMMIT: {
+      ret = COMMIT_BITCODE;
+      break;
+    }
+
+    default: {
+      ret = UNKNOWN_BITCODE;
+      break;
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerOpCond::init(const char *op_cond)
+{
+  int ret = OB_SUCCESS;
+  char *saveptr = nullptr;
+  char *op = nullptr;
+  int64_t arg_len = op_cond == nullptr ? 0 : strlen(op_cond);
+  char *tmp_buffer = static_cast<char*>(ob_malloc(arg_len + 1, "LogMnrOpCond"));
+  if (OB_ISNULL(op_cond)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("ObLogMinerOpCond init with a null op_cond, unexpected", K(op_cond), K(arg_len));
+  } else if (OB_ISNULL(tmp_buffer)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("allocate temp buffer failed", K(tmp_buffer), K(arg_len), K(op_cond));
+  } else {
+    MEMCPY(tmp_buffer, op_cond, arg_len);
+    tmp_buffer[arg_len] = '\0';
+    // ddl, begin, commit, heartbeat need to be processed by default
+    op_cond_ |= (DDL_BITCODE | BEGIN_BITCODE | COMMIT_BITCODE | HEARTBEAT_BITCODE);
+    op = STRTOK_R(tmp_buffer, OP_DELIMITER, &saveptr);
+    while (nullptr != op && OB_SUCC(ret)) {
+      op = str_trim(op);
+      if (0 == strcasecmp(op, INSERT_OP_STR)) {
+        op_cond_ |= INSERT_BITCODE;
+      } else if (0 == strcasecmp(op, UPDATE_OP_STR)) {
+        op_cond_ |= UPDATE_BITCODE;
+      } else if (0 == strcasecmp(op, DELETE_OP_STR)) {
+        op_cond_ |= DELETE_BITCODE;
+      } else {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_ERROR("get unexpected operation", K(op_cond_), K(op_cond), K(op));
+        LOGMINER_STDOUT("parse operations failed\n");
+        break;
+      }
+      op = STRTOK_R(nullptr , OP_DELIMITER, &saveptr);
+    }
+  }
+
+  if (OB_NOT_NULL(tmp_buffer)) {
+    ob_free(tmp_buffer);
+  }
+  return ret;
+}
+
+bool ObLogMinerOpCond::is_record_type_match(const RecordType type) const
+{
+  bool bret = false;
+  const int64_t record_bitcode = record_type_to_bitcode(type);
+  if (record_bitcode & op_cond_) {
+    bret = true;
+  }
+  return bret;
+}
+
+void ObLogMinerOpCond::reset()
+{
+  op_cond_ = 0;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_filter_condition.h

@@ -0,0 +1,188 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ *
+ */
+
+#ifndef OCEANBASE_LOG_MINER_FILTER_CONDITION_H_
+#define OCEANBASE_LOG_MINER_FILTER_CONDITION_H_
+
+#include "lib/json/ob_json.h"
+#include "lib/string/ob_string_buffer.h"
+#include "libobcdc.h"
+#include "lib/container/ob_se_array.h"
+#include "lib/hash/ob_linear_hash_map.h"
+#include "lib/string/ob_fixed_length_string.h"
+#include "ob_log_miner_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+struct ObLogMinerColVal
+{
+  ObLogMinerColVal():
+      col_(),
+      val_(),
+      is_null_(false),
+      allocator_(nullptr) { }
+  ~ObLogMinerColVal() { destroy(); }
+  int init(const ObString &column_name,
+      const ObString &value,
+      const bool is_null,
+      ObIAllocator *alloc);
+  void destroy();
+
+  TO_STRING_KV(K(col_), K(val_));
+
+  ColumnName col_;
+  ObString val_;
+  bool is_null_;
+private:
+  ObIAllocator *allocator_;
+};
+
+typedef ObSEArray<ObLogMinerColVal, 1> ObLogMinerColumnVals;
+struct DbAndTableWrapper
+{
+  DbAndTableWrapper(const char *db_name, const char *tbl_name):
+      db_name_(db_name),
+      table_name_(tbl_name) {}
+
+  int hash(uint64_t &val) const;
+
+  bool operator==(const DbAndTableWrapper &that) const {
+    return db_name_ == that.db_name_ && table_name_ == that.table_name_;
+  }
+
+  TO_STRING_KV(
+    K_(db_name),
+    K_(table_name)
+  );
+
+  DbName    db_name_;
+  TableName table_name_;
+};
+
+// Indicates column conditions of one table
+struct ObLogMinerTableColumnCond
+{
+public:
+  static const char *DATABASE_KEY;
+  static const char *TABLE_KEY;
+  static const char *COLUMN_COND_KEY;
+public:
+  explicit ObLogMinerTableColumnCond():
+      db_name_(), table_name_(), column_conds_(), alloc_(nullptr) { }
+  int init(ObIAllocator *alloc, json::Object &obj);
+  int init(ObIAllocator *alloc, const ObString &db, const ObString &tbl);
+  void destroy();
+
+  uint64_t hash() const;
+  int hash(uint64_t &hash_val) const;
+
+
+  TO_STRING_KV(
+    K_(db_name),
+    K_(table_name),
+    K_(column_conds)
+  );
+
+private:
+  int add_column_cond_(json::Object &obj);
+
+public:
+  DbName    db_name_;
+  TableName table_name_;
+  ObSEArray<ObLogMinerColumnVals, 1> column_conds_;
+private:
+  ObIAllocator *alloc_;
+};
+// expect table cond is a json string with format below:
+// [
+//   {
+//     "database_name":"db1",
+//     "table_name":"tbl1",
+//     "column_cond":[
+//       {
+//         "col1":"val1",
+//         "col2":"val2"
+//       },
+//       {
+//         "col3":"val3",
+//         "col4":null
+//       }
+//     ]
+//   },
+//   {
+//     "database_name":"db1",
+//     "table_name":"tbl2"
+//   }
+// ]
+
+typedef ObLinearHashMap<DbAndTableWrapper, ObLogMinerTableColumnCond*>  TableColumnCondMap;
+
+struct ObLogMinerMultiTableColumnCond
+{
+  explicit ObLogMinerMultiTableColumnCond(ObIAllocator *alloc):
+    table_column_conds_(), alloc_(alloc) { }
+  int init(const char *table_cond_str);
+  void destroy();
+
+  TO_STRING_KV("table_column_conds count ", table_column_conds_.count());
+
+private:
+  int add_table_cond_(json::Object &obj);
+
+  int parse_json_cond_(const char *table_cond_str);
+
+public:
+  TableColumnCondMap table_column_conds_;
+private:
+  ObIAllocator *alloc_;
+};
+
+
+struct ObLogMinerOpCond
+{
+public:
+  // RecordType need to be processsed
+  static const int64_t INSERT_BITCODE = 1 << 0;
+  static const int64_t UPDATE_BITCODE = 1 << 1;
+  static const int64_t DELETE_BITCODE = 1 << 2;
+  static const int64_t DDL_BITCODE = 1 << 3;
+  static const int64_t HEARTBEAT_BITCODE = 1 << 4;
+  static const int64_t BEGIN_BITCODE = 1 << 5;
+  static const int64_t COMMIT_BITCODE = 1 << 6;
+  static const int64_t UNKNOWN_BITCODE = 1 << 63;
+
+  static const char *INSERT_OP_STR;
+  static const char *UPDATE_OP_STR;
+  static const char *DELETE_OP_STR;
+  static const char *OP_DELIMITER;
+
+  static int64_t record_type_to_bitcode(const RecordType type);
+
+public:
+  ObLogMinerOpCond():
+      op_cond_(0) { }
+  int init(const char *op_cond);
+  void reset();
+  bool is_record_type_match(const RecordType type) const;
+  TO_STRING_KV(K_(op_cond));
+public:
+  int64_t op_cond_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_flashback_reader.cpp

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */

src/logservice/logminer/ob_log_miner_flashback_reader.h

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_FLASHBACK_READER_H_
+#define OCEANBASE_LOG_MINER_FLASHBACK_READER_H_
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+class ILogMinerFlashbackReader {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void destroy() = 0;
+};
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_flashback_writer.cpp

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */

src/logservice/logminer/ob_log_miner_flashback_writer.h

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_FLASHBACK_WRITER_H_
+#define OCEANBASE_LOG_MINER_FLASHBACK_WRITER_H_
+
+#include "lib/string/ob_sql_string.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerFlashbackWriter {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void destroy() = 0;
+  virtual int push(common::ObSqlString *sql) = 0;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_logger.cpp

@@ -0,0 +1,104 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include <sys/ioctl.h>
+#include <unistd.h>
+#include "ob_log_miner_logger.h"
+#include "ob_log_miner_timezone_getter.h"
+#include "lib/string/ob_string.h"
+#include "lib/timezone/ob_time_convert.h"             // ObTimeConverter
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+LogMinerLogger &LogMinerLogger::get_logminer_logger_instance()
+{
+  static LogMinerLogger logger_instance;
+  return logger_instance;
+}
+LogMinerLogger::LogMinerLogger():
+    verbose_(false) { memset(pb_str_, '>', sizeof(pb_str_)); }
+
+void LogMinerLogger::log_stdout(const char *format, ...)
+{
+  va_list vl;
+  va_start(vl, format);
+  vfprintf(stdout, format, vl);
+  va_end(vl);
+}
+
+void LogMinerLogger::log_stdout_v(const char *format, ...)
+{
+  if (verbose_) {
+    log_stdout(format);
+  }
+}
+
+int LogMinerLogger::log_progress(int64_t record_num, int64_t current_ts, int64_t begin_ts, int64_t end_ts)
+{
+  int ret = OB_SUCCESS;
+  double percentage = double(current_ts - begin_ts) / double(end_ts - begin_ts);
+  double progress = 0;
+  int pb_width = 0;
+  int terminal_width = 0;
+  int lpad = 0;
+  int rpad = 0;
+  int64_t pos = 0;
+  const ObString nls_format;
+  char time_buf[128] = {0};
+  char pb_buf[MAX_SCREEN_WIDTH] = {0};
+  // current_ts may exceed end_ts
+  if (percentage > 1) {
+    percentage = 1;
+  }
+  progress = percentage * 100;
+  terminal_width = get_terminal_width();
+  terminal_width = (terminal_width < MAX_SCREEN_WIDTH) ? terminal_width : MAX_SCREEN_WIDTH;
+  pb_width = terminal_width - FIXED_TERMINAL_WIDTH;
+  pb_width = (pb_width < MIN_PB_WIDTH) ? 0 : pb_width;
+
+  if (0 < pb_width) {
+    lpad = (int)(percentage * pb_width);
+    rpad = pb_width - lpad;
+    sprintf(pb_buf, "[%.*s%*s]", lpad, pb_str_, rpad, "");
+  }
+  if (OB_FAIL(ObTimeConverter::datetime_to_str(current_ts, &LOGMINER_TZ.get_tz_info(),
+      nls_format, 0, time_buf, sizeof(time_buf), pos))) {
+    LOG_WARN("datetime to string failed", K(current_ts), K(LOGMINER_TZ.get_tz_info()));
+  } else {
+    fprintf(stdout, "\r%s %s %5.1lf%%, written records: %-20jd", time_buf, pb_buf,
+        progress, record_num);
+    fflush(stdout);
+  }
+  return ret;
+}
+
+int LogMinerLogger::get_terminal_width()
+{
+  int tmp_ret = OB_SUCCESS;
+  struct winsize terminal_size {};
+  if (isatty(STDOUT_FILENO)) {
+    if (-1 == ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size)) { // On error, -1 is returned
+      tmp_ret = OB_ERR_UNEXPECTED;
+      LOG_WARN_RET(tmp_ret, "get terminal width failed", K(errno), K(strerror(errno)));
+    }
+  }
+  // if get terminal width failed, return 0.
+  return tmp_ret == OB_SUCCESS ? terminal_size.ws_col : 0;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_logger.h

@@ -0,0 +1,51 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_LOGGER_H_
+#define OCEANBASE_LOG_MINER_LOGGER_H_
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+#define LOGMINER_STDOUT(...) \
+::oceanbase::oblogminer::LogMinerLogger::get_logminer_logger_instance().log_stdout(__VA_ARGS__)
+#define LOGMINER_STDOUT_V(...) \
+::oceanbase::oblogminer::LogMinerLogger::get_logminer_logger_instance().log_stdout_v(__VA_ARGS__)
+#define LOGMINER_LOGGER \
+::oceanbase::oblogminer::LogMinerLogger::get_logminer_logger_instance()
+class LogMinerLogger {
+public:
+  LogMinerLogger();
+  static LogMinerLogger &get_logminer_logger_instance();
+  void set_verbose(bool verbose) {
+    verbose_ = verbose;
+  }
+  void log_stdout(const char *format, ...);
+  void log_stdout_v(const char *format, ...);
+  int log_progress(int64_t record_num, int64_t current_ts, int64_t begin_ts, int64_t end_ts);
+private:
+  int get_terminal_width();
+
+private:
+  static const int MIN_PB_WIDTH = 5;
+  // 68: 20->datatime, 2->[], 7->percentage, 19->", written records: ", 20->the length of INT64_MAX
+  static const int FIXED_TERMINAL_WIDTH = 68;
+  static const int MAX_SCREEN_WIDTH = 4096;
+  bool verbose_;
+  char pb_str_[MAX_SCREEN_WIDTH];
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_main.cpp

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner.h"
+#include "ob_log_miner_args.h"
+#include "ob_log_miner_logger.h"
+#include "lib/oblog/ob_log.h"
+#include "share/rc/ob_tenant_base.h"
+
+using namespace oceanbase::oblogminer;
+using namespace oceanbase::common;
+using namespace oceanbase::share;
+
+int main(int argc, char *argv[])
+{
+  int ret = OB_SUCCESS;
+  ObLogMinerArgs args;
+  ObLogMiner logminer_instance;
+  // set OB_SERVER_TENANT_ID as oblogminer's MTL_ID.
+  // if not set, `ob_malloc` maybe generate error logs
+  // due to `OB_INVALID_TENANT_ID`.
+  ObTenantBase tenant_ctx(OB_SERVER_TENANT_ID);
+  ObTenantEnv::set_tenant(&tenant_ctx);
+
+  OB_LOGGER.set_file_name(ObLogMinerArgs::LOGMINER_LOG_FILE, true, false);
+  OB_LOGGER.set_log_level("WDIAG");
+  OB_LOGGER.set_enable_async_log(false);
+
+  if (OB_FAIL(args.init(argc, argv))) {
+    LOG_ERROR("logminer get invalid arguments", K(argc), K(args));
+    LOGMINER_STDOUT("logminer get invalid arguments, please check log[%s] for more detail\n",
+        ObLogMinerArgs::LOGMINER_LOG_FILE);
+  } else if (args.print_usage_) {
+    ObLogMinerCmdArgs::print_usage(argv[0]);
+  } else if (OB_FAIL(logminer_instance.init(args))) {
+    LOG_ERROR("logminer instance init failed", K(args));
+    LOGMINER_STDOUT("logminer init failed, please check log[%s] for more detail\n",
+        ObLogMinerArgs::LOGMINER_LOG_FILE);
+  } else {
+    logminer_instance.run();
+  }
+
+  return ret;
+}

src/logservice/logminer/ob_log_miner_mode.cpp

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_miner_mode.h"
+#include <cstring>
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+const char *logminer_mode_to_str(LogMinerMode mode)
+{
+  const char *mode_str = nullptr;
+  switch(mode) {
+    case LogMinerMode::UNKNOWN: {
+      mode_str = "UNKNOWN";
+      break;
+    }
+
+    case LogMinerMode::ANALYSIS: {
+      mode_str = "ANALYSIS";
+      break;
+    }
+
+    case LogMinerMode::FLASHBACK: {
+      mode_str = "FLASHBACK";
+      break;
+    }
+
+    case LogMinerMode::MAX_MODE: {
+      mode_str = "MAX_MODE";
+      break;
+    }
+
+    default: {
+      mode_str = "INVALID";
+      break;
+    }
+  }
+
+  return mode_str;
+}
+
+LogMinerMode get_logminer_mode(const common::ObString &mode_str)
+{
+  LogMinerMode mode = LogMinerMode::UNKNOWN;
+  if (0 == mode_str.case_compare("analysis")) {
+    mode = LogMinerMode::ANALYSIS;
+  } else if (0 == mode_str.case_compare("flashback")) {
+    mode = LogMinerMode::FLASHBACK;
+  } else {
+    // return UNKNOWN mode
+  }
+  return mode;
+}
+
+LogMinerMode get_logminer_mode(const char *mode_str)
+{
+  LogMinerMode mode = LogMinerMode::UNKNOWN;
+
+  if (0 == strcasecmp(mode_str, "analysis")) {
+    mode = LogMinerMode::ANALYSIS;
+  } else if (0 == strcasecmp(mode_str, "flashback")) {
+    mode = LogMinerMode::FLASHBACK;
+  } else {
+    // return UNKNOWN mode
+  }
+
+  return mode;
+}
+
+bool is_logminer_mode_valid(const LogMinerMode mode)
+{
+  return mode > LogMinerMode::UNKNOWN && mode < LogMinerMode::MAX_MODE;
+}
+
+bool is_analysis_mode(const LogMinerMode mode)
+{
+  return LogMinerMode::ANALYSIS == mode;
+}
+
+bool is_flashback_mode(const LogMinerMode mode)
+{
+  return LogMinerMode::FLASHBACK == mode;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_mode.h

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_MODE_H_
+#define OCEANBASE_LOG_MINER_MODE_H_
+
+#include "lib/string/ob_string.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+enum class LogMinerMode {
+  UNKNOWN = 0,
+
+  ANALYSIS = 1,
+  FLASHBACK = 2,
+
+  MAX_MODE
+};
+
+const char *logminer_mode_to_str(LogMinerMode mode);
+LogMinerMode get_logminer_mode(const common::ObString &mode_str);
+LogMinerMode get_logminer_mode(const char *mode_str);
+bool is_logminer_mode_valid(const LogMinerMode mode);
+bool is_analysis_mode(const LogMinerMode mode);
+bool is_flashback_mode(const LogMinerMode mode);
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_progress_range.cpp

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_progress_range.h"
+#include "ob_log_miner_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+////////////////////////////// ObLogMinerProgressRange //////////////////////////////
+
+const char *ObLogMinerProgressRange::min_commit_ts_key = "MIN_COMMIT_TS";
+const char *ObLogMinerProgressRange::max_commit_ts_key = "MAX_COMMIT_TS";
+
+int ObLogMinerProgressRange::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n%s=%ld\n",
+      min_commit_ts_key, min_commit_ts_, max_commit_ts_key, max_commit_ts_))) {
+    LOG_ERROR("failed to serialize progress range into buffer", K(buf_len), K(pos), KPC(this));
+  }
+  return ret;
+}
+
+int ObLogMinerProgressRange::deserialize(const char *buf, const int64_t data_len, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(parse_line(min_commit_ts_key, buf, data_len, pos, min_commit_ts_))) {
+    LOG_ERROR("parse line for min_commit_ts failed", K(min_commit_ts_key), K(data_len), K(pos));
+  } else if (OB_FAIL(parse_line(max_commit_ts_key, buf, data_len, pos, max_commit_ts_))) {
+    LOG_ERROR("parse line for max_commit_ts failed", K(max_commit_ts_key), K(data_len), K(pos));
+  }
+  return ret;
+}
+
+int64_t ObLogMinerProgressRange::get_serialize_size() const
+{
+  int64_t size = 0;
+  const int64_t max_digit_size = 30;
+  char digit_str[max_digit_size];
+  size += snprintf(digit_str, max_digit_size, "%ld", min_commit_ts_);
+  size += snprintf(digit_str, max_digit_size, "%ld", max_commit_ts_);
+  // min_commit_ts_key + max_commit_ts_key + '=' + '\n' + '=' + '\n'
+  size += strlen(min_commit_ts_key) + strlen(max_commit_ts_key) + 4;
+  return size;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_progress_range.h

@@ -0,0 +1,68 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_PROGRESS_RANGE_H_
+#define OCEANBASE_LOG_MINER_PROGRESS_RANGE_H_
+
+#include "lib/ob_define.h"
+#include "lib/utility/ob_print_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+struct ObLogMinerProgressRange
+{
+  static const char *min_commit_ts_key;
+  static const char *max_commit_ts_key;
+
+  ObLogMinerProgressRange()
+  { reset(); }
+
+  void reset()
+  {
+    min_commit_ts_ = OB_INVALID_TIMESTAMP;
+    max_commit_ts_ = OB_INVALID_TIMESTAMP;
+  }
+
+  bool is_valid() const
+  {
+    return min_commit_ts_ != OB_INVALID_TIMESTAMP && max_commit_ts_ != OB_INVALID_TIMESTAMP;
+  }
+
+  ObLogMinerProgressRange &operator=(const ObLogMinerProgressRange &that)
+  {
+    min_commit_ts_ = that.min_commit_ts_;
+    max_commit_ts_ = that.max_commit_ts_;
+    return *this;
+  }
+
+  bool operator==(const ObLogMinerProgressRange &that) const
+  {
+    return min_commit_ts_ == that.min_commit_ts_ && max_commit_ts_ == that.max_commit_ts_;
+  }
+
+  TO_STRING_KV(
+    K(min_commit_ts_),
+    K(max_commit_ts_)
+  );
+
+  NEED_SERIALIZE_AND_DESERIALIZE;
+
+  int64_t min_commit_ts_;
+  int64_t max_commit_ts_;
+};
+
+}
+}
+#endif

src/logservice/logminer/ob_log_miner_record.h

@@ -0,0 +1,279 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_H_
+#define OCEANBASE_LOG_MINER_RECORD_H_
+
+#include "lib/string/ob_string_buffer.h"
+#include "lib/container/ob_se_array.h"
+#include "lib/worker.h"
+#include "rpc/obmysql/ob_mysql_global.h"
+#include "storage/tx/ob_trans_define.h"
+
+#include "ob_log_miner_br.h"
+#include "ob_log_miner_recyclable_task.h"
+#include "ob_log_miner_utils.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerBR;
+
+typedef ObSEArray<ObString, 4> KeyArray;
+class ObLogMinerRecord: public ObLogMinerRecyclableTask
+{
+public:
+  ObLogMinerRecord();
+	explicit ObLogMinerRecord(ObIAllocator *alloc);
+	~ObLogMinerRecord();
+    int init(ObLogMinerBR &logminer_br);
+	void set_allocator(ObIAllocator *alloc);
+	void destroy();
+	void reset();
+
+	bool is_inited() { return is_inited_; }
+
+	lib::Worker::CompatMode get_compat_mode() const {
+		return compat_mode_;
+	}
+
+	bool is_mysql_compat_mode() const {
+		return lib::Worker::CompatMode::MYSQL == compat_mode_;
+	}
+
+	bool is_oracle_compat_mode() const {
+		return lib::Worker::CompatMode::ORACLE == compat_mode_;
+	}
+
+	uint64_t get_tenant_id() const {
+		return tenant_id_;
+	}
+
+	int64_t get_cluster_id() const {
+		return orig_cluster_id_;
+	}
+
+	const TenantName& get_tenant_name() const {
+		return tenant_name_;
+	}
+
+	const DbName& get_database_name() const {
+		return database_name_;
+	}
+
+	const TableName& get_table_name() const {
+		return table_name_;
+	}
+
+	const transaction::ObTransID& get_ob_trans_id() const {
+		return trans_id_;
+	}
+
+	const KeyArray& get_primary_keys() const {
+		return primary_keys_;
+	}
+
+	const KeyArray& get_unique_keys() const {
+		return unique_keys_;
+	}
+
+	const ObString& get_row_unique_id() const {
+		return row_unique_id_;
+	}
+
+	RecordType get_record_type() const {
+		return record_type_;
+	}
+
+	share::SCN get_commit_scn() const {
+		return commit_scn_;
+	}
+
+	const ObStringBuffer& get_redo_stmt() const {
+		return redo_stmt_;
+	}
+
+	const ObStringBuffer& get_undo_stmt() const {
+		return undo_stmt_;
+	}
+
+	int build_stmts(ObLogMinerBR &br);
+
+	bool is_dml_record() const;
+	bool is_ddl_record() const;
+
+	void copy_base_info(const ObLogMinerRecord &other);
+
+	TO_STRING_KV(
+		K(is_inited_),
+		K(is_filtered_),
+		K(compat_mode_),
+		K(tenant_id_),
+		K(tenant_name_),
+		K(database_name_),
+		K(table_name_),
+		K(trans_id_),
+		K(primary_keys_),
+		K(unique_keys_),
+		K(record_type_),
+		K(commit_scn_),
+		"redo_stmt_len", redo_stmt_.length(),
+		"undo_stmt_len", undo_stmt_.length()
+	);
+
+private:
+	void free_keys_(KeyArray &arr);
+
+	void free_row_unique_id_();
+
+	int copy_string_to_array_(const ObString &str, KeyArray &array);
+
+	int fill_row_unique_id_(ICDCRecord &cdc_record);
+
+	int fill_data_record_fields_(ICDCRecord &cdc_record);
+
+	int fill_tenant_db_name_(const char *tenant_db_name);
+
+	int fill_primary_keys_(ITableMeta &tbl_meta);
+
+	int fill_unique_keys_(ITableMeta &tbl_meta);
+
+	int fill_keys_(const char *key_cstr, KeyArray &key_arr);
+
+	int build_ddl_stmt_(ICDCRecord &cdc_record);
+
+	int build_dml_stmt_(ICDCRecord &cdc_record);
+
+	int build_insert_stmt_(ObStringBuffer &stmt,
+			binlogBuf *new_cols,
+			const unsigned int new_col_cnt,
+			ITableMeta *tbl_meta);
+
+	int build_insert_stmt_(ObStringBuffer &stmt,
+			binlogBuf *new_cols,
+			const unsigned int new_col_cnt,
+			ITableMeta *tbl_meta,
+			bool &has_lob_null);
+
+	int build_update_stmt_(ObStringBuffer &stmt,
+			binlogBuf *new_cols,
+			const unsigned int new_col_cnt,
+			binlogBuf *old_cols,
+			const unsigned int old_col_cnt,
+			ITableMeta *tbl_meta,
+			bool &has_lob_null,
+			bool &has_unsupport_type_compare);
+
+	int build_delete_stmt_(ObStringBuffer &stmt,
+			binlogBuf *old_cols,
+			const unsigned int old_col_cnt,
+			ITableMeta *tbl_meta,
+			bool &has_lob_null,
+			bool &has_unsupport_type_compare);
+
+	int build_column_value_(ObStringBuffer &stmt,
+			IColMeta *col_meta,
+			binlogBuf &col_data);
+
+	int build_where_conds_(ObStringBuffer &stmt,
+		binlogBuf *cols,
+		const unsigned int col_cnt,
+		ITableMeta *tbl_meta,
+		bool &has_lob_null,
+		bool &has_unsupport_type_compare);
+
+	int build_key_conds_(ObStringBuffer &stmt,
+		binlogBuf *cols,
+		const unsigned int col_cnt,
+		ITableMeta *tbl_meta,
+		const KeyArray &key,
+		bool &has_lob_null,
+		bool &has_unsupport_type_compare);
+
+	int build_cond_(ObStringBuffer &stmt,
+		binlogBuf *cols,
+		const unsigned int col_idx,
+		ITableMeta *tbl_meta,
+		IColMeta *col_meta,
+		bool &has_lob_null,
+		bool &has_unsupport_type_compare);
+
+	int build_lob_cond_(ObStringBuffer &stmt,
+		binlogBuf *cols,
+		const unsigned int col_idx,
+		ITableMeta *tbl_meta,
+		IColMeta *col_meta,
+		bool &has_lob_null,
+		bool &has_unsupport_type_compare);
+
+	int build_func_cond_(ObStringBuffer &stmt,
+		binlogBuf *cols,
+		const unsigned int col_idx,
+		ITableMeta *tbl_meta,
+		IColMeta *col_meta,
+		const char *func_name);
+
+	int build_normal_cond_(ObStringBuffer &stmt,
+		binlogBuf *cols,
+		const unsigned int col_idx,
+		ITableMeta *tbl_meta,
+		IColMeta *col_meta);
+
+	int build_hex_val_(ObStringBuffer &stmt,
+			binlogBuf &col_data);
+
+	int build_escape_char_(ObStringBuffer &stmt);
+	bool is_string_type_(IColMeta *col_meta) const;
+
+	bool is_binary_type_(IColMeta *col_meta) const;
+
+	bool is_number_type_(IColMeta *col_meta) const;
+
+	bool is_lob_type_(IColMeta *col_meta) const;
+
+	bool is_geo_type_(IColMeta *col_meta) const;
+
+	bool is_bit_type_(IColMeta *col_meta) const;
+
+private:
+    static const char *ORACLE_ESCAPE_CHAR;
+	static const char *MYSQL_ESCAPE_CHAR;
+	static const char *ORA_GEO_PREFIX;
+	static const char *JSON_EQUAL;
+	static const char *LOB_COMPARE;
+	static const char *ST_EQUALS;
+
+	bool 					is_inited_;
+	bool					is_filtered_;
+	ObIAllocator			*alloc_;
+
+	lib::Worker::CompatMode	compat_mode_;
+	uint64_t		        tenant_id_;
+	int64_t					orig_cluster_id_;
+	TenantName				tenant_name_;
+	DbName					database_name_;
+	TableName		        table_name_;
+	transaction::ObTransID	trans_id_;
+	KeyArray				primary_keys_;
+	KeyArray				unique_keys_;
+	ObString				row_unique_id_;
+	RecordType				record_type_;
+	share::SCN 				commit_scn_;
+	ObStringBuffer 			redo_stmt_;
+	ObStringBuffer 			undo_stmt_;
+};
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_record_aggregator.cpp

@@ -0,0 +1,255 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_record_aggregator.h"
+#include "ob_log_miner_batch_record_writer.h"
+#include "ob_log_miner_batch_record.h"
+#include "ob_log_miner_record.h"
+#include "ob_log_miner_data_manager.h"
+#include "ob_log_miner_resource_collector.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+const int64_t ObLogMinerRecordAggregator::RECORD_AGG_THREAD_NUM = 1L;
+const int64_t ObLogMinerRecordAggregator::RECORD_AGG_QUEUE_SIZE = 100000L;
+const int64_t ObLogMinerRecordAggregator::RECORD_FLUSH_THRESHOLD = 1L * 1000 * 1000; // 1 sec
+
+ObLogMinerRecordAggregator::ObLogMinerRecordAggregator():
+  is_inited_(false),
+  cur_batch_record_(nullptr),
+  push_record_count_(0),
+  aggregated_record_count_(0),
+  last_trans_end_ts_(OB_INVALID_TIMESTAMP),
+  writer_(nullptr),
+  data_manager_(nullptr),
+  resource_collector_(nullptr),
+  err_handle_(nullptr) { }
+
+ObLogMinerRecordAggregator::~ObLogMinerRecordAggregator()
+{
+  destroy();
+}
+
+int ObLogMinerRecordAggregator::init(const int64_t start_time_us,
+    ILogMinerBatchRecordWriter *writer,
+    ILogMinerDataManager *data_manager,
+    ILogMinerResourceCollector *resource_collector,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("ObLogMinerRecord Aggregator has been initialized", K(is_inited_));
+  } else if (OB_ISNULL(writer) || OB_ISNULL(data_manager) || OB_ISNULL(resource_collector)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("ObLogMinerRecord Aggregator get invalid argument", K(writer), K(data_manager));
+  } else if (OB_FAIL(RecordAggThreadPool::init(RECORD_AGG_THREAD_NUM, RECORD_AGG_QUEUE_SIZE))) {
+    LOG_ERROR("RecordAggThreadPool failed to init");
+  } else {
+    writer_ = writer;
+    data_manager_ = data_manager;
+    resource_collector_ = resource_collector;
+    err_handle_ = err_handle;
+    push_record_count_ = 0;
+    aggregated_record_count_ = 0;
+    last_trans_end_ts_ = start_time_us;
+    is_inited_ = true;
+    LOG_INFO("ObLogMinerRecordAggregator finished to init", K(writer_), K(data_manager_));
+  }
+
+  return ret;
+}
+
+int ObLogMinerRecordAggregator::start()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerRecord Aggregator hasn't been initialized", K(is_inited_));
+  } else if (OB_FAIL(RecordAggThreadPool::start())) {
+    LOG_ERROR("RecordAggThreadPool failed to start");
+  } else {
+    LOG_INFO("ObLogMinerRecordAggregator starts");
+  }
+  return ret;
+}
+
+void ObLogMinerRecordAggregator::stop()
+{
+  RecordAggThreadPool::mark_stop_flag();
+  LOG_INFO("ObLogMinerRecordAggregator stopped");
+}
+
+void ObLogMinerRecordAggregator::wait()
+{
+  RecordAggThreadPool::stop();
+  LOG_INFO("ObLogMinerRecordAggregator wait end");
+}
+
+void ObLogMinerRecordAggregator::destroy()
+{
+  if (IS_INIT) {
+    if (nullptr != cur_batch_record_ && nullptr != data_manager_) {
+      data_manager_->release_logminer_batch_record(cur_batch_record_);
+    }
+    RecordAggThreadPool::destroy();
+    cur_batch_record_ = nullptr;
+    writer_ = nullptr;
+    data_manager_ = nullptr;
+    resource_collector_ = nullptr;
+    err_handle_ = nullptr;
+    is_inited_ = false;
+  }
+}
+
+int ObLogMinerRecordAggregator::push(ObLogMinerRecord *record)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("aggregator has not been initialized", K(is_inited_));
+  } else if (OB_ISNULL(record)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid record in aggregator", K(record));
+  } else {
+    LOG_TRACE("ObLogMinerRecordAggregator push record into queue", KPC(record));
+    const int64_t DATA_OP_TIMEOUT = 1L * 1000 * 1000;
+    int64_t hash_val = 0;
+    RETRY_FUNC(is_stoped(), *(static_cast<ObMQThread*>(this)), push, record, hash_val, DATA_OP_TIMEOUT);
+    if (OB_FAIL(ret)) {
+      LOG_ERROR("push record into MQThread queue failed", KPC(record), "is_stopped", is_stoped());
+    } else {
+      ATOMIC_INC(&push_record_count_);
+    }
+  }
+  return ret;
+}
+
+int ObLogMinerRecordAggregator::handle(void *data, const int64_t thread_index, volatile bool &stop_flag)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("aggregator has not been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid record in aggregator", K(data));
+  } else {
+    ObLogMinerRecord *record = static_cast<ObLogMinerRecord*>(data);
+    bool need_flush = false;
+    LOG_TRACE("aggregator process record", KPC(record));
+
+    if (is_trans_end_record_type(record->get_record_type())) {
+      last_trans_end_ts_ = record->get_commit_scn().convert_to_ts();
+    }
+
+    if (nullptr == cur_batch_record_) {
+      if (OB_FAIL(data_manager_->get_logminer_batch_record(cur_batch_record_))) {
+        LOG_ERROR("get batch record from data manager failed", KPC(cur_batch_record_));
+      } else if (OB_ISNULL(cur_batch_record_)) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_ERROR("get null batch record from data_manager", KPC(cur_batch_record_));
+      } else if (OB_FAIL(cur_batch_record_->init_last_trans_end_ts(last_trans_end_ts_))) {
+        LOG_ERROR("failed to init last trans end ts", KPC(cur_batch_record_));
+      }
+    }
+
+    if (OB_FAIL(ret)) {
+      LOG_ERROR("failed to get cur_batch_record");
+    } else if (OB_FAIL(cur_batch_record_->append_record(*record))) {
+      LOG_ERROR("append record to cur_batch_record failed", KPC(record), KPC(cur_batch_record_));
+    } else if (OB_FAIL(check_need_flush_(need_flush))) {
+      LOG_ERROR("check need flush failed", KPC(cur_batch_record_));
+    } else if (OB_FAIL(resource_collector_->revert(record))) {
+      LOG_ERROR("resource collector failed to revert record", KP(record));
+    } else {
+      LOG_TRACE("batch_record need flush", K(need_flush));
+      if (need_flush) {
+        ATOMIC_AAF(&aggregated_record_count_, cur_batch_record_->get_total_record_count());
+        if (OB_FAIL(writer_->push(cur_batch_record_))) {
+          LOG_ERROR("push cur_batch_record into writer failed", KPC(cur_batch_record_));
+        } else {
+          cur_batch_record_ = nullptr;
+        }
+      }
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "ObLogMinerRecordAggregator exit unexpected\n");
+  }
+
+  return ret;
+}
+
+int ObLogMinerRecordAggregator::get_total_record_count(int64_t &record_count)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("aggregator has not been initialized", K(is_inited_));
+  } else {
+    record_count = ATOMIC_LOAD(&push_record_count_) - ATOMIC_LOAD(&aggregated_record_count_);
+  }
+
+  return ret;
+}
+
+int ObLogMinerRecordAggregator::check_need_flush_(bool &need_flush)
+{
+  int ret = OB_SUCCESS;
+  const char *reason = "INVALID";
+  int64_t first_access_ts = OB_INVALID_TIMESTAMP;
+  need_flush = false;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("aggregator has not been initialized", K(is_inited_));
+  } else if (OB_ISNULL(cur_batch_record_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("cur_batch_record_ is null when check_need_flush", K(cur_batch_record_));
+  } else if (cur_batch_record_->is_freezed()) {
+    need_flush = true;
+    reason = "Freezed";
+  } else if (! cur_batch_record_->get_progress_range().is_valid()) {
+    need_flush = false;
+    reason = "Invalid Progress Range";
+  } else if (data_manager_->reach_end_progress(cur_batch_record_->get_last_trans_end_ts())) {
+    need_flush = true;
+    reason = "Reach End Progress";
+    LOG_INFO("cur_bacth_record reached end progress", KPC(cur_batch_record_), K(need_flush));
+  } else {
+    first_access_ts = cur_batch_record_->get_first_access_ts();
+    if (OB_INVALID_TIMESTAMP == first_access_ts) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("cur_batch_record hasn't been accessed when check_need_flush", KPC(cur_batch_record_));
+    } else if (ObTimeUtility::current_time() - first_access_ts >= RECORD_FLUSH_THRESHOLD) {
+      need_flush = true;
+      reason = "Time Threshold";
+    } else {
+      reason = "Wait Flush";
+    }
+  }
+
+  LOG_TRACE("check_need_flush end", KR(ret), K(need_flush), K(reason), K(first_access_ts));
+
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_record_aggregator.h

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_AGGREGATOR_H_
+#define OCEANBASE_LOG_MINER_RECORD_AGGREGATOR_H_
+
+#include "lib/thread/ob_multi_fixed_queue_thread.h"
+#include "ob_log_miner_error_handler.h"
+#include "ob_log_miner_batch_record.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerRecord;
+class ILogMinerAnalysisWriter;
+class ILogMinerDataManager;
+class ILogMinerBatchRecordWriter;
+class ILogMinerResourceCollector;
+
+class ILogMinerRecordAggregator
+{
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int push(ObLogMinerRecord *record) = 0;
+  virtual int get_total_record_count(int64_t &record_count) = 0;
+};
+
+typedef common::ObMQThread<1, ILogMinerRecordAggregator> RecordAggThreadPool;
+
+class ObLogMinerRecordAggregator: public ILogMinerRecordAggregator, public RecordAggThreadPool
+{
+public:
+  static const int64_t RECORD_AGG_THREAD_NUM;
+  static const int64_t RECORD_AGG_QUEUE_SIZE;
+  static const int64_t RECORD_FLUSH_THRESHOLD;
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int push(ObLogMinerRecord *record);
+  virtual int get_total_record_count(int64_t &record_count);
+public:
+  virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag);
+
+public:
+  ObLogMinerRecordAggregator();
+  ~ObLogMinerRecordAggregator();
+
+  int init(const int64_t start_time_us,
+      ILogMinerBatchRecordWriter *writer,
+      ILogMinerDataManager *data_manager,
+      ILogMinerResourceCollector *resource_collector,
+      ILogMinerErrorHandler *err_handle);
+
+private:
+  int check_need_flush_(bool &need_flush);
+
+private:
+  bool is_inited_;
+  ObLogMinerBatchRecord *cur_batch_record_;
+  int64_t push_record_count_ CACHE_ALIGNED;
+  int64_t aggregated_record_count_ CACHE_ALIGNED;
+  int64_t last_trans_end_ts_;
+  ILogMinerBatchRecordWriter *writer_;
+  ILogMinerDataManager *data_manager_;
+  ILogMinerResourceCollector *resource_collector_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_record_converter.cpp

@@ -0,0 +1,445 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "lib/json_type/ob_json_base.h"               // ObJsonBaseUtil
+#include "lib/timezone/ob_time_convert.h"             // ObTimeConverter
+#include "ob_log_miner_record_converter.h"
+#include "logservice/common_util/ob_log_time_utils.h"
+#include "ob_log_miner_logger.h"
+#include "ob_log_miner_utils.h"
+
+#define APPEND_STR(buf, args...) \
+  do {\
+    if (OB_SUCC(ret) && OB_FAIL(buf.append(args))) { \
+      LOG_ERROR("append str failed", "buf_len", buf.length(), "buf_cap", buf.capacity()); \
+    }\
+  } while(0)
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+#define MINER_SCHEMA_DEF(field, id, args...) \
+ILogMinerRecordConverter::ColType::field,
+const ILogMinerRecordConverter::ColType ILogMinerRecordConverter::COL_ORDER[] =
+{
+  #include "ob_log_miner_analyze_schema.h"
+};
+#undef MINER_SCHEMA_DEF
+
+const char *ILogMinerRecordConverter::DELIMITER = ",";
+
+ILogMinerRecordConverter *ILogMinerRecordConverter::get_converter_instance(const RecordFileFormat format)
+{
+  static ObLogMinerRecordCsvConverter csv_converter;
+  static ObLogMinerRecordRedoSqlConverter redo_sql_converter;
+  static ObLogMinerRecordUndoSqlConverter undo_sql_converter;
+  static ObLogMinerRecordJsonConverter json_converter;
+
+  ILogMinerRecordConverter *converter = nullptr;
+  switch(format) {
+    case RecordFileFormat::CSV: {
+      converter = &csv_converter;
+      break;
+    }
+
+    case RecordFileFormat::REDO_ONLY: {
+      converter = &redo_sql_converter;
+      break;
+    }
+
+    case RecordFileFormat::UNDO_ONLY: {
+      converter = &undo_sql_converter;
+      break;
+    }
+
+    case RecordFileFormat::JSON: {
+      converter = &json_converter;
+      break;
+    }
+
+    default: {
+      converter = nullptr;
+      break;
+    }
+  }
+
+  return converter;
+}
+
+////////////////////////////// ObLogMinerRecordCsvConverter //////////////////////////////
+
+int ObLogMinerRecordCsvConverter::write_record(const ObLogMinerRecord &record,
+    common::ObStringBuffer &buffer, bool &is_written)
+{
+  int ret = OB_SUCCESS;
+
+  const int64_t col_num = sizeof(COL_ORDER) / sizeof(ColType);
+
+  for (int64_t i = 0; i < col_num && OB_SUCC(ret); i++) {
+    const char *end_char = i == col_num-1 ? "\n": DELIMITER;
+    switch(COL_ORDER[i]) {
+      case ColType::TENANT_ID: {
+        if (OB_FAIL(write_unsigned_number(record.get_tenant_id(), buffer))) {
+          LOG_ERROR("write tenant_id failed", K(record));
+        } // tenant_id
+        break;
+      }
+
+      case ColType::TRANS_ID: {
+        if (OB_FAIL(write_signed_number(record.get_ob_trans_id().get_id(), buffer))) {
+          LOG_ERROR("write trans_id failed", K(record));
+        } // trans_id
+        break;
+      }
+
+      case ColType::PRIMARY_KEY: {
+        if (OB_FAIL(write_keys(record.get_primary_keys(), buffer))) {
+          LOG_ERROR("write primary_key failed", K(record));
+        } // primary_key
+        break;
+      }
+
+      case ColType::TENANT_NAME: {
+        if (OB_FAIL(write_string_no_escape(record.get_tenant_name().str(), buffer))) {
+          LOG_ERROR("write tenant_name failed", K(record));
+        } // tenant_name
+        break;
+      }
+
+      case ColType::DATABASE_NAME: {
+        if (OB_FAIL(write_string_no_escape(record.get_database_name().str(), buffer))) {
+          LOG_ERROR("write database_name failed", K(record));
+        } // database_name
+        break;
+      }
+
+      case ColType::TABLE_NAME: {
+        if (OB_FAIL(write_string_no_escape(record.get_table_name().str(), buffer))) {
+          LOG_ERROR("write table_name failed", K(record));
+        } // table_name
+        break;
+      }
+
+      case ColType::OPERATION: {
+        if (OB_FAIL(write_string_no_escape(record_type_to_str(record.get_record_type()), buffer))) {
+          LOG_ERROR("write operation failed", K(record));
+        } // operation
+        break;
+      }
+
+      case ColType::OPERATION_CODE: {
+        if (OB_FAIL(write_signed_number(record_type_to_num(record.get_record_type()), buffer))) {
+          LOG_ERROR("write operation_code failed", K(record));
+        } // operation_code
+        break;
+      }
+
+      case ColType::COMMIT_SCN: {
+        if (OB_FAIL(write_signed_number(record.get_commit_scn().get_val_for_inner_table_field(), buffer))) {
+          LOG_ERROR("write commit_scn failed", K(record));
+        } // commit scn
+        break;
+      }
+
+      case ColType::COMMIT_TIMESTAMP: {
+        const int16_t scale = 6;
+        char time_buf[128] = {0};
+        int64_t pos = 0;
+        ObString nls_format;
+        if (OB_FAIL(ObTimeConverter::datetime_to_str(record.get_commit_scn().convert_to_ts(),
+            &LOGMINER_TZ.get_tz_info(), nls_format, scale, time_buf, sizeof(time_buf), pos))) {
+          LOG_ERROR("failed to get time string from commit_scn", K(record));
+        } else if (OB_FAIL(write_string_no_escape(time_buf, buffer))) {
+          LOG_ERROR("write commit_timestamp failed", K(record));
+        }
+        break;
+      }
+
+      case ColType::SQL_REDO: {
+        if (OB_FAIL(write_csv_string_escape_(record.get_redo_stmt().string(), buffer))) {
+          LOG_ERROR("write redo_stmt failed", K(record));
+        } // redo_stmt
+        break;
+      }
+
+      case ColType::SQL_UNDO: {
+        if (OB_FAIL(write_csv_string_escape_(record.get_undo_stmt().string(), buffer))) {
+          LOG_ERROR("write undo_stmt failed", K(record));
+        } // undo_stmt
+        break;
+      }
+
+      case ColType::ORG_CLUSTER_ID: {
+        if (OB_FAIL(write_signed_number(record.get_cluster_id(), buffer))) {
+          LOG_ERROR("write org_cluster_id failed", K(record));
+        } // org_cluster_id
+        break;
+      }
+
+      default: {
+        ret = OB_ERR_DEFENSIVE_CHECK;
+        LOG_ERROR("unsupported column type", K(COL_ORDER[i]), K(record));
+        break;
+      }
+    }
+
+    if (OB_SUCC(ret) && OB_FAIL(buffer.append(end_char))) {
+      LOG_ERROR("append delimiter failed", K(record), K(end_char));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    is_written = true;
+    LOG_TRACE("ObLogMinerRecordCsvConverter write record succ", K(record));
+  }
+
+  return ret;
+}
+
+int ObLogMinerRecordCsvConverter::write_csv_string_escape_(const ObString &str, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  const char *data = str.ptr(), *prev_ptr = data;
+  APPEND_STR(buffer, "\"");
+  while (OB_SUCC(ret) && nullptr != prev_ptr && nullptr != (data = strchr(prev_ptr, '"'))) {
+    APPEND_STR(buffer, prev_ptr, data - prev_ptr + 1);
+    APPEND_STR(buffer, "\"");
+    prev_ptr = data + 1;
+  }
+
+  APPEND_STR(buffer, prev_ptr);
+  APPEND_STR(buffer, "\"");
+  return ret;
+}
+
+////////////////////////////// ObLogMinerRecordRedoSqlConverter //////////////////////////////
+
+int ObLogMinerRecordRedoSqlConverter::write_record(const ObLogMinerRecord &record,
+    common::ObStringBuffer &buffer, bool &is_written)
+{
+  int ret = OB_SUCCESS;
+  if (!record.get_redo_stmt().empty()) {
+    APPEND_STR(buffer, record.get_redo_stmt().string());
+    APPEND_STR(buffer, "\n");
+    if (OB_SUCC(ret)) {
+      is_written = true;
+      LOG_TRACE("ObLogMinerRecordRedoSqlConverter write record succ", K(record));
+    }
+  }
+  return ret;
+}
+
+////////////////////////////// ObLogMinerRecordUndoSqlConverter //////////////////////////////
+
+int ObLogMinerRecordUndoSqlConverter::write_record(const ObLogMinerRecord &record,
+    common::ObStringBuffer &buffer, bool &is_written)
+{
+  int ret = OB_SUCCESS;
+  if (!record.get_undo_stmt().empty()) {
+    APPEND_STR(buffer, record.get_undo_stmt().string());
+    APPEND_STR(buffer, "\n");
+    if (OB_SUCC(ret)) {
+      is_written = true;
+      LOG_TRACE("ObLogMinerRecordUndoSqlConverter write record succ", K(record));
+    }
+  }
+  return ret;
+}
+
+////////////////////////////// ObLogMinerRecordJsonConverter //////////////////////////////
+
+int ObLogMinerRecordJsonConverter::write_record(const ObLogMinerRecord &record,
+    common::ObStringBuffer &buffer, bool &is_written)
+{
+  int ret = OB_SUCCESS;
+
+  const int64_t col_num = sizeof(COL_ORDER) / sizeof(ColType);
+  APPEND_STR(buffer, "{");
+
+  for (int64_t i = 0; i < col_num && OB_SUCC(ret); i++) {
+    const char *end_char = i == col_num-1 ? "}\n": DELIMITER;
+    switch(COL_ORDER[i]) {
+      case ColType::TENANT_ID: {
+        if (OB_FAIL(write_json_key_("TENANT_ID", buffer))) {
+          LOG_ERROR("write json_key TENANT_ID failed", K(record));
+        } else if (OB_FAIL(write_unsigned_number(record.get_tenant_id(), buffer))) {
+          LOG_ERROR("write tenant_id failed", K(record));
+        } // tenant_id
+        break;
+      }
+
+      case ColType::TRANS_ID: {
+        if (OB_FAIL(write_json_key_("TRANS_ID", buffer))) {
+          LOG_ERROR("write json_key TRANS_ID failed", K(record));
+        } else if (OB_FAIL(write_signed_number(record.get_ob_trans_id().get_id(), buffer))) {
+          LOG_ERROR("write trans_id failed", K(record));
+        } // trans_id
+        break;
+      }
+
+      case ColType::PRIMARY_KEY: {
+        if (OB_FAIL(write_json_key_("PRIMARY_KEY", buffer))) {
+          LOG_ERROR("write json_key PRIMARY_KEY failed", K(record));
+        } else if (OB_FAIL(write_keys(record.get_primary_keys(), buffer))) {
+          LOG_ERROR("write primary_key failed", K(record));
+        } // primary_key
+        break;
+      }
+
+      case ColType::TENANT_NAME: {
+        if (OB_FAIL(write_json_key_("TENANT_NAME", buffer))) {
+          LOG_ERROR("write json_key TENANT_NAME failed", K(record));
+        } else if (OB_FAIL(write_string_no_escape(record.get_tenant_name().str(), buffer))) {
+          LOG_ERROR("write tenant_name failed", K(record));
+        } // tenant_name
+        break;
+      }
+
+      case ColType::DATABASE_NAME: {
+        if (OB_FAIL(write_json_key_("DATABASE_NAME", buffer))) {
+          LOG_ERROR("write json_key DATABASE_NAME failed", K(record));
+        } else if (OB_FAIL(write_string_no_escape(record.get_database_name().str(), buffer))) {
+          LOG_ERROR("write database_name failed", K(record));
+        } // database_name
+        break;
+      }
+
+      case ColType::TABLE_NAME: {
+        if (OB_FAIL(write_json_key_("TABLE_NAME", buffer))) {
+          LOG_ERROR("write json_key TABLE_NAME failed", K(record));
+        } else if (OB_FAIL(write_string_no_escape(record.get_table_name().str(), buffer))) {
+          LOG_ERROR("write table_name failed", K(record));
+        } // table_name
+        break;
+      }
+
+      case ColType::OPERATION: {
+        if (OB_FAIL(write_json_key_("OPERATION", buffer))) {
+          LOG_ERROR("write json_key OPERATION failed", K(record));
+        } else if (OB_FAIL(write_string_no_escape(record_type_to_str(record.get_record_type()), buffer))) {
+          LOG_ERROR("write operation failed", K(record));
+        } // operation
+        break;
+      }
+
+      case ColType::OPERATION_CODE: {
+        if (OB_FAIL(write_json_key_("OPERATION_CODE", buffer))) {
+          LOG_ERROR("write json_key OPERATION_CODE failed", K(record));
+        } else if (OB_FAIL(write_signed_number(record_type_to_num(record.get_record_type()), buffer))) {
+          LOG_ERROR("write operation_code failed", K(record));
+        } // operation_code
+        break;
+      }
+
+      case ColType::COMMIT_SCN: {
+        if (OB_FAIL(write_json_key_("COMMIT_SCN", buffer))) {
+          LOG_ERROR("write json_key COMMIT_SCN failed", K(record));
+        } else if (OB_FAIL(write_signed_number(record.get_commit_scn().get_val_for_inner_table_field(), buffer))) {
+          LOG_ERROR("write commit_scn failed", K(record));
+        } // commit scn
+        break;
+      }
+
+      case ColType::COMMIT_TIMESTAMP: {
+        const int16_t scale = 6;
+        char time_buf[128] = {0};
+        int64_t pos = 0;
+        ObString nls_format;
+        if (OB_FAIL(write_json_key_("COMMIT_TIMESTAMP", buffer))) {
+          LOG_ERROR("write json_key COMMIT_TIMESTAMP failed", K(record));
+        } else if (OB_FAIL(ObTimeConverter::datetime_to_str(record.get_commit_scn().convert_to_ts(),
+            &LOGMINER_TZ.get_tz_info(), nls_format, scale, time_buf, sizeof(time_buf), pos))) {
+          LOG_ERROR("failed to get time string from commit_scn", K(record));
+        } else if (OB_FAIL(write_string_no_escape(time_buf, buffer))) {
+          LOG_ERROR("write commit_timestamp failed", K(record));
+        }
+        break;
+      }
+
+      case ColType::SQL_REDO: {
+        if (OB_FAIL(write_json_key_("SQL_REDO", buffer))) {
+          LOG_ERROR("write json_key SQL_REDO failed", K(record));
+        } else if (OB_FAIL(write_json_string_escape_(record.get_redo_stmt().string(), buffer))) {
+          LOG_ERROR("write redo_stmt failed", K(record));
+        } // redo_stmt
+        break;
+      }
+
+      case ColType::SQL_UNDO: {
+        if (OB_FAIL(write_json_key_("SQL_UNDO", buffer))) {
+          LOG_ERROR("write json_key SQL_UNDO failed", K(record));
+        } else if (OB_FAIL(write_json_string_escape_(record.get_undo_stmt().string(), buffer))) {
+          LOG_ERROR("write undo_stmt failed", K(record));
+        } // undo_stmt
+        break;
+      }
+
+      case ColType::ORG_CLUSTER_ID: {
+        if (OB_FAIL(write_json_key_("ORG_CLUSTER_ID", buffer))) {
+          LOG_ERROR("write json_key ORG_CLUSTER_ID failed", K(record));
+        } else if (OB_FAIL(write_signed_number(record.get_cluster_id(), buffer))) {
+          LOG_ERROR("write org_cluster_id failed", K(record));
+        } // org_cluster_id
+        break;
+      }
+
+      default: {
+        ret = OB_ERR_DEFENSIVE_CHECK;
+        LOG_ERROR("unsupported column type", K(COL_ORDER[i]), K(record));
+        break;
+      }
+    }
+
+    if (OB_SUCC(ret) && OB_FAIL(buffer.append(end_char))) {
+      LOG_ERROR("append delimiter failed", K(record), K(end_char));
+    }
+
+    if (OB_SUCC(ret)) {
+      is_written = true;
+      LOG_TRACE("ObLogMinerRecordJsonConverter write record succ", K(record));
+    }
+  }
+
+  return ret;
+}
+
+int ObLogMinerRecordJsonConverter::write_json_key_(const char *str, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  APPEND_STR(buffer, "\"");
+  APPEND_STR(buffer, str);
+  APPEND_STR(buffer, "\":");
+  return ret;
+}
+
+int ObLogMinerRecordJsonConverter::write_json_string_escape_(const ObString &str, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  const char *data = str.ptr();
+  if (nullptr != data) {
+    if (OB_FAIL(ObJsonBaseUtil::add_double_quote(buffer, data, str.length()))) {
+      LOG_ERROR("add_double_quote failed", K(str));
+    }
+  } else {
+    APPEND_STR(buffer, "\"\"");
+  }
+
+  return ret;
+}
+
+}
+}
+
+#undef APPEND_STR

src/logservice/logminer/ob_log_miner_record_converter.h

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_CONVERTER_H_
+#define OCEANBASE_LOG_MINER_RECORD_CONVERTER_H_
+
+#include "ob_log_miner_file_manager.h"
+#include "ob_log_miner_record.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerRecordConverter
+{
+public:
+  // TODO: there may be concurrency issue in future?
+  static ILogMinerRecordConverter *get_converter_instance(const RecordFileFormat format);
+  virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written) = 0;
+
+public:
+  // TENANT_ID,TRANS_ID,PRIMARY_KEY,ROW_UNIQUE_ID,SEQ_NO,TENANT_NAME,USER_NAME,TABLE_NAME,OPERATION,
+  // OPERATION_CODE,COMMIT_SCN,COMMIT_TIMESTAMP,SQL_REDO,SQL_UNDO,ORG_CLUSTER_ID
+  #define MINER_SCHEMA_DEF(field, id, args...) \
+    field = id,
+  enum class ColType {
+    #include "ob_log_miner_analyze_schema.h"
+  };
+  #undef MINER_SCHEMA_DEF
+
+  const static ColType COL_ORDER[];
+  static const char *DELIMITER;
+};
+
+class ObLogMinerRecordCsvConverter: public ILogMinerRecordConverter
+{
+public:
+  virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written);
+public:
+  ObLogMinerRecordCsvConverter() {};
+  ~ObLogMinerRecordCsvConverter() {}
+private:
+  int write_csv_string_escape_(const ObString &str, common::ObStringBuffer &buffer);
+};
+
+class ObLogMinerRecordRedoSqlConverter: public ILogMinerRecordConverter
+{
+public:
+  virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written);
+public:
+  ObLogMinerRecordRedoSqlConverter() {}
+  ~ObLogMinerRecordRedoSqlConverter() {}
+
+};
+
+class ObLogMinerRecordUndoSqlConverter: public ILogMinerRecordConverter
+{
+public:
+  virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written);
+public:
+  ObLogMinerRecordUndoSqlConverter() {}
+  ~ObLogMinerRecordUndoSqlConverter() {}
+};
+
+class ObLogMinerRecordJsonConverter: public ILogMinerRecordConverter
+{
+public:
+  virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written);
+public:
+  ObLogMinerRecordJsonConverter() {};
+  ~ObLogMinerRecordJsonConverter() {}
+private:
+  int write_json_key_(const char *str, common::ObStringBuffer &buffer);
+  int write_json_string_escape_(const ObString &str, common::ObStringBuffer &buffer);
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_record_file_format.cpp

@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_miner_record_file_format.h"
+#include <cstring>
+namespace oceanbase
+{
+namespace oblogminer
+{
+RecordFileFormat get_record_file_format(const common::ObString &file_format_str)
+{
+  RecordFileFormat format = RecordFileFormat::INVALID;
+  if (0 == file_format_str.case_compare("CSV")) {
+    format = RecordFileFormat::CSV;
+  } else if (0 == file_format_str.case_compare("REDO_ONLY")) {
+    format = RecordFileFormat::REDO_ONLY;
+  } else if (0 == file_format_str.case_compare("UNDO_ONLY")) {
+    format = RecordFileFormat::UNDO_ONLY;
+  } else if (0 == file_format_str.case_compare("JSON")) {
+    format = RecordFileFormat::JSON;
+  } else if (0 == file_format_str.case_compare("AVRO")) {
+    format = RecordFileFormat::AVRO;
+  } else if (0 == file_format_str.case_compare("PARQUET")) {
+    format = RecordFileFormat::PARQUET;
+  }
+  return format;
+}
+
+RecordFileFormat get_record_file_format(const char *file_format_str)
+{
+  return get_record_file_format(common::ObString(file_format_str));
+}
+const char *record_file_format_str(const RecordFileFormat format) {
+  const char *result = nullptr;
+  switch (format) {
+    case RecordFileFormat::INVALID:
+      result = "INVALID";
+      break;
+    case RecordFileFormat::CSV:
+      result = "CSV";
+      break;
+    case RecordFileFormat::REDO_ONLY:
+      result = "REDO_ONLY";
+      break;
+    case RecordFileFormat::UNDO_ONLY:
+      result = "UNDO_ONLY";
+      break;
+    case RecordFileFormat::JSON:
+      result = "JSON";
+      break;
+    case RecordFileFormat::PARQUET:
+      result = "PARQUET";
+      break;
+    case RecordFileFormat::AVRO:
+      result = "AVRO";
+      break;
+    default:
+      result = "NOT_SUPPORTED";
+      break;
+  }
+  return result;
+}
+}
+}

src/logservice/logminer/ob_log_miner_record_file_format.h

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_FILE_FORMAT_H_
+#define OCEANBASE_LOG_MINER_RECORD_FILE_FORMAT_H_
+
+#include "lib/string/ob_string.h"
+namespace oceanbase
+{
+namespace oblogminer
+{
+enum class RecordFileFormat
+{
+  INVALID = -1,
+  CSV = 0,
+  REDO_ONLY,
+  UNDO_ONLY,
+  JSON,
+  PARQUET,
+  AVRO
+};
+RecordFileFormat get_record_file_format(const common::ObString &file_format_str);
+
+RecordFileFormat get_record_file_format(const char *file_format_str);
+
+const char *record_file_format_str(const RecordFileFormat format);
+}
+}
+#endif

src/logservice/logminer/ob_log_miner_record_filter.cpp

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */

src/logservice/logminer/ob_log_miner_record_filter.h

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_FILTER_H_
+#define OCEANBASE_LOG_MINER_RECORD_FILTER_H_
+
+#include "lib/string/ob_string.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerRecord;
+
+class ILogMinerRecordFilter {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void destroy() = 0;
+  virtual int push(const ObLogMinerRecord &record) = 0;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_record_parser.cpp

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */

src/logservice/logminer/ob_log_miner_record_parser.h

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_PARSER_H_
+#define OCEANBASE_LOG_MINER_RECORD_PARSER_H_
+
+#include "lib/string/ob_string.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ILogMinerRecordParser {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void destroy() = 0;
+  virtual int push(const common::ObString &record_str) = 0;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_record_rewriter.cpp

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */

src/logservice/logminer/ob_log_miner_record_rewriter.h

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECORD_REWRITER_H_
+#define OCEANBASE_LOG_MINER_RECORD_REWRITER_H_
+
+#include "lib/string/ob_string.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerUndoTask;
+
+class ILogMinerRecordRewriter {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void destroy() = 0;
+  virtual int push(ObLogMinerUndoTask *task) = 0;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_recyclable_task.h

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RECYCLABLE_TASK_H_
+#define OCEANBASE_LOG_MINER_RECYCLABLE_TASK_H_
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerRecyclableTask
+{
+public:
+  enum class TaskType
+  {
+    UNKNOWN = 0,
+    BINLOG_RECORD,
+    LOGMINER_RECORD,
+    BATCH_RECORD,
+    UNDO_TASK
+  };
+
+  explicit ObLogMinerRecyclableTask(TaskType type): type_(type) { }
+  ~ObLogMinerRecyclableTask() { type_ = TaskType::UNKNOWN; }
+
+  bool is_binlog_record() const {
+    return TaskType::BINLOG_RECORD == type_;
+  }
+  bool is_logminer_record() const {
+    return TaskType::LOGMINER_RECORD == type_;
+  }
+  bool is_batch_record() const {
+    return TaskType::BATCH_RECORD == type_;
+  }
+  bool is_undo_task() const {
+    return TaskType::UNDO_TASK == type_;
+  }
+
+  TaskType get_task_type() const
+  {
+    return type_;
+  }
+
+protected:
+  TaskType type_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_resource_collector.cpp

@@ -0,0 +1,218 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_resource_collector.h"
+#include "ob_log_miner_recyclable_task.h"
+#include "ob_log_miner_br.h"
+#include "ob_log_miner_record.h"
+#include "ob_log_miner_batch_record.h"
+#include "ob_log_miner_data_manager.h"
+#include "ob_log_miner_logger.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+const int64_t ObLogMinerResourceCollector::RC_THREAD_NUM = 1L;
+const int64_t ObLogMinerResourceCollector::RC_QUEUE_SIZE = 100000L;
+
+ObLogMinerResourceCollector::ObLogMinerResourceCollector():
+    is_inited_(false),
+    data_manager_(nullptr),
+    err_handle_(nullptr)
+{
+}
+
+ObLogMinerResourceCollector::~ObLogMinerResourceCollector()
+{
+  destroy();
+}
+
+int ObLogMinerResourceCollector::init(ILogMinerDataManager *data_manager,
+    ILogMinerErrorHandler *err_handle)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_ERROR("Resource Collector has been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data_manager)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get invalid argument when trying to initialize resource collector", K(data_manager));
+  } else if (OB_FAIL(ResourceCollectorThread::init(RC_THREAD_NUM, RC_QUEUE_SIZE))) {
+    LOG_ERROR("failed to init resource collector thread", K(RC_THREAD_NUM), K(RC_QUEUE_SIZE));
+  } else {
+    is_inited_ = true;
+    data_manager_ = data_manager;
+    err_handle_ = err_handle_;
+    LOG_INFO("ObLogMinerResourceCollector finished to init");
+    LOGMINER_STDOUT_V("ObLogMinerResourceCollector finished to init\n");
+  }
+
+  return ret;
+}
+
+int ObLogMinerResourceCollector::start()
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_));
+  } else if (OB_FAIL(ResourceCollectorThread::start())) {
+    LOG_ERROR("ResourceCollectorThread failed to start");
+  } else {
+    LOG_INFO("ObLogMinerResourceCollector starts");
+  }
+
+  return ret;
+}
+
+void ObLogMinerResourceCollector::stop()
+{
+  ResourceCollectorThread::mark_stop_flag();
+  LOG_INFO("ObLogMinerResourceCollector stopped");
+}
+
+void ObLogMinerResourceCollector::wait()
+{
+  ResourceCollectorThread::stop();
+  LOG_INFO("ObLogMinerResourceCollector finished to wait");
+}
+
+void ObLogMinerResourceCollector::destroy()
+{
+  if (IS_INIT) {
+    is_inited_ = false;
+    data_manager_ = nullptr;
+    err_handle_ = nullptr;
+    LOG_INFO("ObLogMinerResourceCollector destroyed");
+    LOGMINER_STDOUT_V("ObLogMinerResourceCollector destroyed\n");
+  }
+}
+
+int ObLogMinerResourceCollector::revert(ObLogMinerRecyclableTask *task)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_));
+  } else if (OB_FAIL(push_task_(task))) {
+    LOG_ERROR("failed to push task into thread queue", K(task));
+  }
+  return ret;
+}
+
+int ObLogMinerResourceCollector::get_total_task_count(int64_t &task_count)
+{
+  int ret = OB_SUCCESS;
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_));
+  } else if (OB_FAIL(ResourceCollectorThread::get_total_task_num(task_count))) {
+    LOG_ERROR("failed to get total task count in resource collector");
+  }
+
+  return ret;
+}
+
+int ObLogMinerResourceCollector::handle(void *data, const int64_t thread_index, volatile bool &stop_flag)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_));
+  } else if (OB_ISNULL(data)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("get null data when handling data", K(data));
+  } else {
+    ObLogMinerRecyclableTask *task = static_cast<ObLogMinerRecyclableTask*>(data);
+    if (task->is_logminer_record()) {
+      ObLogMinerRecord *rec = static_cast<ObLogMinerRecord*>(task);
+      if (OB_FAIL(handle_logminer_record_(rec))) {
+        LOG_ERROR("failed to handle logminer record", KPC(rec));
+      }
+    } else if (task->is_binlog_record()) {
+      ObLogMinerBR *br = static_cast<ObLogMinerBR*>(task);
+      if (OB_FAIL(handle_binlog_record_(br))) {
+        LOG_ERROR("failed to handle binlog record", KPC(br));
+      }
+    } else if (task->is_batch_record()) {
+      ObLogMinerBatchRecord *batch_rec = static_cast<ObLogMinerBatchRecord*>(task);
+      if (OB_FAIL(handle_batch_record_(batch_rec))) {
+        LOG_ERROR("failed to handle batch record", KPC(batch_rec));
+      }
+    } else {
+      ret = OB_NOT_SUPPORTED;
+      LOG_ERROR("get unsupported task", "type", task->get_task_type());
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    err_handle_->handle_error(ret, "ObLogMinerResourceCollector exit unexpected\n");
+  }
+  return ret;
+}
+
+int ObLogMinerResourceCollector::push_task_(ObLogMinerRecyclableTask *task)
+{
+  int ret = OB_SUCCESS;
+  const int64_t DATA_OP_TIMEOUT = 1L * 1000 * 1000;
+  static int64_t seq_no = 0;
+  int64_t hash_val = ATOMIC_AAF(&seq_no, 1);
+  RETRY_FUNC(is_stoped(), *(static_cast<ObMQThread*>(this)), push, task, hash_val, DATA_OP_TIMEOUT);
+  return ret;
+}
+
+int ObLogMinerResourceCollector::handle_binlog_record_(ObLogMinerBR *br)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(br)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get null br in resource collector", K(br));
+  } else {
+    br->destroy();
+    ob_free(br);
+  }
+  return ret;
+}
+
+int ObLogMinerResourceCollector::handle_logminer_record_(ObLogMinerRecord *logminer_rec)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(logminer_rec)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get null logminer_rec in resource collector", K(logminer_rec));
+  } else if (OB_FAIL(data_manager_->release_log_miner_record(logminer_rec))) {
+    LOG_ERROR("failed to release log miner record", KPC(logminer_rec));
+  }
+  return ret;
+}
+
+int ObLogMinerResourceCollector::handle_batch_record_(ObLogMinerBatchRecord *batch_rec)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(batch_rec)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("get null batch record in resource collector", K(batch_rec));
+  } else if (OB_FAIL(data_manager_->release_logminer_batch_record(batch_rec))) {
+    LOG_ERROR("failed to release batch record", KPC(batch_rec));
+  }
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_resource_collector.h

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_RESOURCE_COLLECTOR_H_
+#define OCEANBASE_LOG_MINER_RESOURCE_COLLECTOR_H_
+
+#include "lib/thread/ob_multi_fixed_queue_thread.h"
+#include "ob_log_miner_error_handler.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerBR;
+class ObLogMinerRecord;
+class ObLogMinerUndoTask;
+class ObLogMinerBatchRecord;
+class ObLogMinerRecyclableTask;
+class ILogMinerBRProducer;
+class ILogMinerDataManager;
+
+class ILogMinerResourceCollector {
+public:
+  virtual int start() = 0;
+  virtual void stop() = 0;
+  virtual void wait() = 0;
+  virtual void destroy() = 0;
+  virtual int revert(ObLogMinerRecyclableTask *undo_task) = 0;
+  virtual int get_total_task_count(int64_t &task_count) = 0;
+};
+
+typedef common::ObMQThread<1, ILogMinerResourceCollector> ResourceCollectorThread;
+
+class ObLogMinerResourceCollector: public ILogMinerResourceCollector, ResourceCollectorThread
+{
+public:
+  static const int64_t RC_THREAD_NUM;
+  static const int64_t RC_QUEUE_SIZE;
+public:
+  virtual int start();
+  virtual void stop();
+  virtual void wait();
+  virtual void destroy();
+  virtual int revert(ObLogMinerRecyclableTask *undo_task);
+  virtual int get_total_task_count(int64_t &task_count);
+
+public:
+  virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag);
+
+public:
+  ObLogMinerResourceCollector();
+  ~ObLogMinerResourceCollector();
+  int init(ILogMinerDataManager *data_manager,
+      ILogMinerErrorHandler *err_handle);
+
+private:
+  int push_task_(ObLogMinerRecyclableTask *task);
+  int handle_binlog_record_(ObLogMinerBR *br);
+  int handle_logminer_record_(ObLogMinerRecord *logminer_rec);
+  int handle_batch_record_(ObLogMinerBatchRecord *batch_rec);
+
+private:
+  bool is_inited_;
+  ILogMinerDataManager *data_manager_;
+  ILogMinerErrorHandler *err_handle_;
+};
+
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_timezone_getter.cpp

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_timezone_getter.h"
+#include "lib/string/ob_string.h"
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+ObLogMinerTimeZoneGetter &ObLogMinerTimeZoneGetter::get_instance()
+{
+  static ObLogMinerTimeZoneGetter tz_getter_instance;
+  return tz_getter_instance;
+}
+
+ObLogMinerTimeZoneGetter::ObLogMinerTimeZoneGetter():
+    tz_info_() { }
+
+int ObLogMinerTimeZoneGetter::set_timezone(const char *timezone)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(tz_info_.set_timezone(ObString(timezone)))) {
+    LOG_ERROR("parse timezone failed", K(ret), KCSTRING(timezone));
+  }
+  return ret;
+}
+
+}
+}

src/logservice/logminer/ob_log_miner_timezone_getter.h

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_TIMEZONE_GETTER_H_
+#define OCEANBASE_LOG_MINER_TIMEZONE_GETTER_H_
+
+#include "lib/timezone/ob_timezone_info.h"            // ObTimeZoneInfo
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+#define LOGMINER_TZ \
+::oceanbase::oblogminer::ObLogMinerTimeZoneGetter::get_instance()
+
+class ObLogMinerTimeZoneGetter {
+public:
+  static ObLogMinerTimeZoneGetter &get_instance();
+
+  ObLogMinerTimeZoneGetter();
+
+  int set_timezone(const char *timezone);
+
+  const ObTimeZoneInfo &get_tz_info() const {
+    return tz_info_;
+  }
+
+private:
+  ObTimeZoneInfo tz_info_;
+};
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_undo_task.cpp

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */

src/logservice/logminer/ob_log_miner_undo_task.h

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_UNDO_TASK_H_
+#define OCEANBASE_LOG_UNDO_TASK_H_
+
+#include "lib/string/ob_sql_string.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+class ObLogMinerRecord;
+
+class ObLogMinerUndoTask {
+public:
+  ObLogMinerUndoTask();
+  int init(ObLogMinerRecord *logminer_rec);
+
+private:
+  bool                is_filtered_;
+  int64_t             stmt_timestamp_;
+  common::ObSqlString *undo_stmt_;
+};
+}
+}
+
+#endif

src/logservice/logminer/ob_log_miner_utils.cpp

@@ -0,0 +1,458 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX LOGMNR
+
+#include "ob_log_miner_utils.h"
+#include "lib/ob_define.h"
+#include "lib/function/ob_function.h"               // ObFunction
+#include <cerrno>
+
+#define APPEND_STR(buf, args...) \
+  do {\
+    if (OB_SUCC(ret) && OB_FAIL(buf.append(args))) { \
+      LOG_ERROR("append str failed", "buf_len", buf.length(), "buf_cap", buf.capacity()); \
+    }\
+  } while(0)
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+// not very safe, str must end with '\0'
+int logminer_str2ll(const char *str, char* &endptr, int64_t &num)
+{
+  int ret = OB_SUCCESS;
+  if (nullptr == str) {
+    ret = OB_INVALID_ARGUMENT;
+  } else {
+    int origin_errno = errno;
+    errno = 0;
+    num = strtoll(str, &endptr, 10);
+    if (0 != errno || endptr == str) {
+      ret = OB_INVALID_ARGUMENT;
+    }
+    errno = origin_errno;
+  }
+
+  return ret;
+}
+
+int logminer_str2ll(const char *str, int64_t &num)
+{
+  char *endptr = nullptr;
+  return logminer_str2ll(str, endptr, num);
+}
+
+int write_keys(const KeyArray &key_arr, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  int64_t total_size = 0;
+  char *tmp_buffer = nullptr;
+  const char *keys_delimiter = "/";
+  APPEND_STR(buffer, "\"");
+  ARRAY_FOREACH_N(key_arr, idx, key_num) {
+    const ObString &key = key_arr.at(idx);
+    APPEND_STR(buffer, key);
+    if (idx != key_num - 1) {
+      APPEND_STR(buffer, keys_delimiter);
+    }
+  }
+  APPEND_STR(buffer, "\"");
+
+  return ret;
+}
+
+int write_signed_number(const int64_t num, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  const int64_t MAX_NUM_LEN = 30;
+  char buf[MAX_NUM_LEN] = {0};
+  int len = snprintf(buf, MAX_NUM_LEN, "%ld", num);
+  APPEND_STR(buffer, buf, len);
+  return ret;
+}
+
+int write_unsigned_number(const uint64_t num, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  const int64_t MAX_NUM_LEN = 30;
+  char buf[MAX_NUM_LEN] = {0};
+  int len = snprintf(buf, MAX_NUM_LEN, "%lu", num);
+  APPEND_STR(buffer, buf, len);
+  return ret;
+}
+
+int write_string_no_escape(const ObString &str, common::ObStringBuffer &buffer)
+{
+  int ret = OB_SUCCESS;
+  APPEND_STR(buffer, "\"");
+  APPEND_STR(buffer, str);
+  APPEND_STR(buffer, "\"");
+  return ret;
+}
+
+bool is_number(const char *str)
+{
+  bool bool_ret = true;
+  if (nullptr == str) {
+    return false;
+  }
+  for (int64_t i = 0; true == bool_ret; ++i) {
+    if ('\0' == str[i]) {
+      break;
+    } else if (!isdigit(str[i])) {
+      bool_ret = false;
+    }
+  }
+	return bool_ret;
+}
+
+const char *empty_str_wrapper(const char *str)
+{
+  return nullptr == str ? "" : str;
+}
+
+const char *record_type_to_str(const RecordType type)
+{
+  static const char *str = "UNKNOWN";
+
+  switch (type)
+  {
+    case EDELETE:
+      str = "DELETE";
+      break;
+
+    case EINSERT:
+      str = "INSERT";
+      break;
+
+    case EUPDATE:
+      str = "UPDATE";
+      break;
+
+    case EBEGIN:
+      str = "BEGIN";
+      break;
+
+    case ECOMMIT:
+      str = "COMMIT";
+      break;
+
+    case EDDL:
+      str = "DDL";
+      break;
+
+    default:
+      str = "NOT_SUPPORTED";
+      break;
+  }
+
+  return str;
+}
+
+int64_t record_type_to_num(const RecordType type)
+{
+  int num = 0;
+  switch (type)
+  {
+    case EINSERT:
+      num = 1;
+      break;
+
+    case EUPDATE:
+      num = 2;
+      break;
+
+    case EDELETE:
+      num = 3;
+      break;
+
+    case EDDL:
+      num = 4;
+      break;
+
+    case EBEGIN:
+      num = 5;
+      break;
+
+    case ECOMMIT:
+      num = 6;
+      break;
+
+    default:
+      num = 0;
+      break;
+  }
+
+  return num;
+
+}
+
+bool is_trans_end_record_type(const RecordType type)
+{
+  bool bret = false;
+  switch (type) {
+    case EDDL:
+    case ECOMMIT:
+    case HEARTBEAT:
+      bret = true;
+      break;
+    default:
+      bret = false;
+      break;
+  }
+  return bret;
+}
+
+int parse_line(const char *key_str,
+    const char *buf,
+    const int64_t buf_len,
+    int64_t &pos,
+    int64_t &data)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(expect_token(buf, buf_len, pos, key_str))) {
+    LOG_ERROR("failed to get key_str from buf", K(buf_len), K(pos), K(key_str));
+  } else if (OB_FAIL(expect_token(buf, buf_len, pos, "="))) {
+    LOG_ERROR("failed to get symbol '='", K(buf), K(buf_len), K(pos));
+  } else if (OB_FAIL(parse_int(buf, buf_len, pos, data))) {
+    LOG_ERROR("failed to parse int from buf", K(buf), K(buf_len), K(pos));
+  } else if (OB_FAIL(expect_token(buf, buf_len, pos, "\n"))) {
+    LOG_ERROR("failed to get token \\n", K(buf_len), K(pos));
+  }
+
+  return ret;
+}
+
+int parse_int(const char *buf, const int64_t data_len, int64_t &pos, int64_t &data)
+{
+  int ret = OB_SUCCESS;
+  enum class ParseState {
+    BLANK_READ,
+    SYMBOL_READ,
+    DIGIT_READ
+  };
+  ParseState state = ParseState::BLANK_READ;
+  bool positive = true;
+  bool done = false;
+  int64_t orig_pos = pos;
+  data = 0;
+
+  while (pos < data_len && !done && OB_SUCC(ret)) {
+    char c = buf[pos];
+    switch(state) {
+      case ParseState::BLANK_READ: {
+        if (isblank(c)) {
+        } else if (isdigit(c)) {
+          state = ParseState::DIGIT_READ;
+          data = todigit(c);
+        } else if ('+' == c) {
+          state = ParseState::SYMBOL_READ;
+          positive = true;
+        } else if ('-' == c) {
+          state = ParseState::SYMBOL_READ;
+          positive = false;
+        } else {
+          ret = OB_INVALID_DATA;
+        }
+        break;
+      }
+
+      case ParseState::SYMBOL_READ: {
+        if (isdigit(c)) {
+          state = ParseState::DIGIT_READ;
+          if (positive) {
+            data = todigit(c);
+          } else {
+            data = -(todigit(c));
+          }
+        } else {
+          ret = OB_INVALID_DATA;
+        }
+        break;
+      }
+
+      case ParseState::DIGIT_READ: {
+        if (isdigit(c)) {
+          int digit = todigit(c);
+          if (positive) {
+            if (data > INT64_MAX/10) {
+              ret = OB_SIZE_OVERFLOW;
+            } else if (INT64_MAX/10 == data) {
+              if (digit > INT64_MAX % 10) {
+                ret = OB_SIZE_OVERFLOW;
+              } else {
+                data = data * 10 + digit;
+              }
+            } else {
+              data = data * 10 + digit;
+            }
+          } else {
+            if (data < INT64_MIN / 10) {
+              ret = OB_SIZE_OVERFLOW;
+            } else if (data == INT64_MIN/10) {
+              if (digit > -((INT64_MIN % 10 + 10) % 10 - 10)) {
+                ret = OB_SIZE_OVERFLOW;
+              } else {
+                data = data * 10 - digit;
+              }
+            } else {
+              data = data * 10 - digit;
+            }
+          }
+        } else {
+          done = true;
+        }
+        break;
+      }
+    }
+
+    if (!done && OB_SUCC(ret)) {
+      pos++;
+    }
+
+  }
+
+  if (OB_FAIL(ret)) {
+    pos = orig_pos;
+  }
+
+
+  return ret;
+}
+
+int parse_line(const char *key_str,
+    const char *buf,
+    const int64_t buf_len,
+    int64_t &pos,
+    ObIAllocator &alloc,
+    char *&value)
+{
+  int ret = OB_SUCCESS;
+  ObString tmp_str;
+  if (OB_FAIL(parse_line(key_str, buf, buf_len, pos, tmp_str))) {
+    LOG_ERROR("failed to parse line to tmp_str", K(buf_len), K(pos), K(key_str));
+  } else if (OB_FAIL(ob_dup_cstring(alloc, tmp_str, value))) {
+    LOG_ERROR("failed to dup cstring to value", K(tmp_str));
+  }
+  return ret;
+}
+
+int parse_line(const char *key_str,
+    const char *buf,
+    const int64_t buf_len,
+    int64_t &pos,
+    ObString &value)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(expect_token(buf, buf_len, pos, key_str))) {
+    LOG_ERROR("failed to get key_str from buf", K(buf_len), K(pos), K(key_str));
+  } else if (OB_FAIL(expect_token(buf, buf_len, pos, "="))) {
+    LOG_ERROR("failed to get symbol '='", K(buf), K(buf_len), K(pos));
+  } else if (OB_FAIL(parse_string(buf, buf_len, pos, value))) {
+    LOG_ERROR("failed to parse string from buf", K(buf), K(buf_len), K(pos));
+  } else if (OB_FAIL(expect_token(buf, buf_len, pos, "\n"))) {
+    LOG_ERROR("failed to get token \\n", K(buf_len), K(pos));
+  }
+  return ret;
+}
+
+int parse_string(const char *buf, const int64_t data_len, int64_t &pos, ObString &value)
+{
+  int ret = OB_SUCCESS;
+  const char *end_ptr = static_cast<const char*>(memchr(buf + pos, '\n', data_len - pos));
+  if (nullptr == end_ptr) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("cannot find the end of line in buf", K(pos), K(data_len));
+  } else {
+    const int64_t value_len = end_ptr - (buf + pos);
+    value.assign_ptr(buf + pos, value_len);
+    pos += value_len;
+  }
+  return ret;
+}
+
+int expect_token(const char *buf, const int64_t data_len, int64_t &pos, const char *token)
+{
+  int ret = OB_SUCCESS;
+
+  int64_t token_len = strlen(token);
+
+  if (pos + token_len > data_len) {
+    ret = OB_SIZE_OVERFLOW;
+  } else if (0 != memcmp(buf + pos, token, token_len)) {
+    ret = OB_INVALID_DATA;
+  } else {
+    pos += token_len;
+  }
+
+  return ret;
+}
+
+int deep_copy_cstring(ObIAllocator &alloc, const char *src_str, char *&dst_str)
+{
+  int ret = OB_SUCCESS;
+
+  if (nullptr != dst_str) {
+    ret = OB_ERR_UNEXPECTED;
+  } else if (nullptr == src_str) {
+    // do nothing
+  } else {
+    const int64_t src_str_len = strlen(src_str);
+    if (OB_ISNULL(dst_str = static_cast<char*>(alloc.alloc(src_str_len + 1)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+    } else {
+      MEMCPY(dst_str, src_str, src_str_len);
+      dst_str[src_str_len] = '\0';
+    }
+  }
+
+  return ret;
+}
+
+int uint_to_bit(const uint64_t bit_uint, ObStringBuffer &bit_str)
+{
+  int ret = OB_SUCCESS;
+  const int bits = 64;
+  int i = bits-1;
+  if (0 == bit_uint) {
+    if (OB_FAIL(bit_str.append("0"))) {
+      LOG_ERROR("bit_str append failed", K(bit_str));
+    }
+  } else {
+    // ignore leading zeroes
+    while (0 == (bit_uint & (1ull << i)) && 0 <= i) {
+      i--;
+    }
+    while (OB_SUCC(ret) && 0 <= i) {
+      if (OB_FAIL(bit_str.append((bit_uint & (1ull << i)) ? "1" : "0"))) {
+        LOG_ERROR("bit_str append failed", K(bit_str));
+      }
+      i--;
+    }
+  }
+  return ret;
+}
+
+int todigit(char c)
+{
+  return c - '0';
+}
+
+
+
+}
+}
+
+#undef APPEND_STR

src/logservice/logminer/ob_log_miner_utils.h

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_UTILS_H_
+#define OCEANBASE_LOG_MINER_UTILS_H_
+
+#include "lib/alloc/alloc_struct.h"
+#include "lib/allocator/ob_malloc.h"
+#include "lib/container/ob_se_array.h"          //ObSEArray
+#include "lib/string/ob_fixed_length_string.h"
+#include "lib/string/ob_string_buffer.h"
+#include "libobcdc.h"
+#ifndef OB_USE_DRCMSG
+#include "ob_cdc_msg_convert.h"
+#endif
+#include <cstdint>
+namespace oceanbase
+{
+namespace oblogminer
+{
+typedef common::ObFixedLengthString<common::OB_MAX_DATABASE_NAME_LENGTH> DbName;
+typedef common::ObFixedLengthString<common::OB_MAX_TABLE_NAME_LENGTH> TableName;
+typedef common::ObFixedLengthString<common::OB_MAX_COLUMN_NAME_LENGTH> ColumnName;
+typedef common::ObFixedLengthString<common::OB_MAX_TENANT_NAME_LENGTH> TenantName;
+typedef common::ObSEArray<ObString, 4> KeyArray;
+
+int logminer_str2ll(const char *str, char* &endptr, int64_t &num);
+int logminer_str2ll(const char *str, int64_t &num);
+
+int write_keys(const KeyArray &key_arr, common::ObStringBuffer &buffer);
+int write_signed_number(const int64_t num, common::ObStringBuffer &buffer);
+int write_unsigned_number(const uint64_t num, common::ObStringBuffer &buffer);
+int write_string_no_escape(const ObString &str, common::ObStringBuffer &buffer);
+
+bool is_number(const char *str);
+
+const char *empty_str_wrapper(const char *str);
+
+const char *record_type_to_str(const RecordType type);
+int64_t record_type_to_num(const RecordType type);
+int parse_line(const char *key_str, const char *buf, const int64_t buf_len, int64_t &pos, int64_t &data);
+int parse_int(const char *buf, const int64_t data_len, int64_t &pos, int64_t &data);
+// allocate memory for value, which is a standard cstring
+int parse_line(const char *key_str,
+    const char *buf,
+    const int64_t buf_len,
+    int64_t &pos,
+    ObIAllocator &alloc,
+    char *&value);
+// no memory alloc, direct assign contents in buffer to value
+int parse_line(const char *key_str,
+    const char *buf,
+    const int64_t buf_len,
+    int64_t &pos,
+    ObString &value);
+// int parse_string(const char *buf, const int64_t data_len, int64_t &pos, ObIAllocator &alloc, char *&value);
+int parse_string(const char *buf, const int64_t data_len, int64_t &pos, ObString &value);
+int expect_token(const char *buf, const int64_t data_len, int64_t &pos, const char *token);
+
+int deep_copy_cstring(ObIAllocator &alloc, const char *src_str, char *&dst_str);
+
+bool is_trans_end_record_type(const RecordType type);
+
+int uint_to_bit(const uint64_t bit_val, ObStringBuffer &str_val);
+
+int todigit(char c);
+
+template<class Derived, class Base, class ...Args>
+int init_component(Base *&ptr, Args&& ...arg)
+{
+  int ret = OB_SUCCESS;
+  lib::ObMemAttr attr(OB_SERVER_TENANT_ID, "CompnentAlloc");
+  Derived *tmp_var = OB_NEW(Derived, attr);
+  const char *derived_name = typeid(Derived).name();
+  const char *base_name = typeid(Base).name();
+  if (OB_ISNULL(tmp_var)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOGMNR_LOG(ERROR, "allocate memory for component failed", K(tmp_var),
+        K(derived_name), K(base_name));
+  } else if (OB_FAIL(tmp_var->init(arg...))) {
+    LOGMNR_LOG(ERROR, "init tmp_var failed", K(tmp_var),
+      K(derived_name), K(base_name));
+  } else {
+    ptr = tmp_var;
+    LOGMNR_LOG(INFO, "init component finished", K(derived_name), K(base_name));
+  }
+  return ret;
+}
+
+template<class Derived, class Base>
+void destroy_component(Base *&ptr)
+{
+  if (OB_NOT_NULL(ptr)) {
+    const char *derived_name = typeid(Derived).name();
+    const char *base_name = typeid(Base).name();
+    Derived *tmp_ptr = static_cast<Derived*>(ptr);
+    tmp_ptr->destroy();
+    OB_DELETE(Derived, "ComponentFree", tmp_ptr);
+    ptr = nullptr;
+    LOGMNR_LOG(INFO, "finished to destroy component", K(derived_name), K(base_name));
+  }
+}
+}
+}
+
+#endif

unittest/CMakeLists.txt

@@ -32,3 +32,4 @@ add_subdirectory(share)
 add_subdirectory(rootserver)
 add_subdirectory(tools)
 add_subdirectory(data_dictionary)
+add_subdirectory(logminer)

unittest/logminer/CMakeLists.txt

@@ -0,0 +1,27 @@
+set(UTIL_FILE_NAME ob_log_miner_test_utils.cpp)
+
+function(logminer_unittest case)
+  if(ARGC EQUAL 1)
+    add_executable(${case} ${case}.cpp ${UTIL_FILE_NAME})
+  else()
+    add_executable(${ARGV} ${UTIL_FILE_NAME})
+  endif()
+  if (case MATCHES "^test_.*")
+    add_test(${case} ${case})
+    set_tests_properties(${case} PROPERTIES TIMEOUT 300)
+  endif()
+  target_link_libraries(${case} PRIVATE oblogminer_obj_dev gtest gmock)
+  target_include_directories(${case}
+    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/unittest ${CMAKE_SOURCE_DIR}/deps/oblib/unittest)
+endfunction()
+
+logminer_unittest(test_ob_log_miner_args)
+logminer_unittest(test_ob_log_miner_br_filter)
+logminer_unittest(test_ob_log_miner_record)
+logminer_unittest(test_ob_log_miner_record_converter)
+logminer_unittest(test_ob_log_miner_utils)
+logminer_unittest(test_ob_log_miner_analyzer_checkpoint)
+logminer_unittest(test_ob_log_miner_progress_range)
+logminer_unittest(test_ob_log_miner_file_meta)
+logminer_unittest(test_ob_log_miner_file_index)
+logminer_unittest(test_ob_log_miner_file_manager)

unittest/logminer/ob_log_miner_test_utils.cpp

@@ -0,0 +1,186 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_binlog_record.h"
+#include "ob_log_miner_br.h"
+#include "ob_log_miner_test_utils.h"
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+ObLogMinerBR *v_build_logminer_br(binlogBuf *new_bufs,
+              binlogBuf *old_bufs,
+              RecordType type,
+              lib::Worker::CompatMode compat_mode,
+              const char *db_name,
+              const char *table_name,
+              const char *encoding,
+              const int arg_count, va_list *ap)
+{
+  ObLogMinerBR *br = new ObLogMinerBR;
+  EXPECT_NE(nullptr, br);
+  libobcdc::ObLogBR *cdc_br = new libobcdc::ObLogBR;
+  cdc_br->init_data(EINSERT, 0, 1, 0, ObString(), ObString(), ObString(),
+      100, 100);
+  ICDCRecord *rec = DRCMessageFactory::createBinlogRecord();
+  rec->setUserData(cdc_br);
+  EXPECT_NE(nullptr, rec);
+  ITableMeta *tab_meta = DRCMessageFactory::createTableMeta();
+  EXPECT_NE(nullptr, tab_meta);
+  IDBMeta *db_meta = DRCMessageFactory::createDBMeta();
+  EXPECT_NE(nullptr, db_meta);
+  rec->setNewColumn(new_bufs, arg_count/3);
+  rec->setOldColumn(old_bufs, arg_count/3);
+  rec->setRecordType(type);
+  rec->setDbname(db_name);
+  rec->setTbname(table_name);
+  db_meta->setName(db_name);
+  tab_meta->setPKs("");
+  tab_meta->setPkinfo("");
+  tab_meta->setUKs("");
+  tab_meta->setUkinfo("");
+  tab_meta->setName(table_name);
+  tab_meta->setDBMeta(db_meta);
+  IColMeta *col_meta = nullptr;
+  const char *next = nullptr;
+  int64_t arg_idx = 0;
+  for (int i = 0; i < arg_count; i++) {
+    const char *col_arg = nullptr;
+    int len = 0;
+    int data_type = 0;
+    if (3 > i%4) {
+      col_arg = va_arg(*ap, const char *);
+      len = col_arg == nullptr ? 0 : strlen(col_arg);
+    } else {
+      data_type = va_arg(*ap, int);
+    }
+    if (0 == i%4) { // column name
+      col_meta = DRCMessageFactory::createColMeta();
+      EXPECT_NE(col_meta, nullptr);
+      col_meta->setName(col_arg);
+      col_meta->setEncoding(encoding);
+      tab_meta->append(col_arg, col_meta);
+    } else if (1 == i%4 && EDELETE != type) { // new value, EDELETE hasn't new value
+      rec->putNew(col_arg, len);
+    } else if (2 == i%4 && EINSERT != type) { // old value, EINSERT hasn't old value
+      rec->putOld(col_arg, len);
+    } else if (3 == i%4) { // column data type
+      col_meta->setType(data_type);
+    }
+  }
+  rec->setTableMeta(tab_meta);
+  br->init(nullptr, rec, compat_mode, 0, 1, 0);
+  return br;
+}
+
+ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs,
+              binlogBuf *old_bufs,
+              RecordType type,
+              lib::Worker::CompatMode compat_mode,
+              const char *db_name,
+              const char *table_name,
+              const char *encoding,
+              const int arg_count, ...)
+{
+  va_list ap;
+  ObLogMinerBR *br = nullptr;
+  va_start(ap, arg_count);
+  br = v_build_logminer_br(new_bufs, old_bufs, type,
+      compat_mode, db_name, table_name, encoding, arg_count, &ap);
+  va_end(ap);
+  return br;
+}
+
+ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs,
+              binlogBuf *old_bufs,
+              RecordType type,
+              lib::Worker::CompatMode compat_mode,
+              const char *db_name,
+              const char *table_name,
+              const int arg_count, ...)
+{
+  va_list ap;
+  ObLogMinerBR *br = nullptr;
+  va_start(ap, arg_count);
+  br = v_build_logminer_br(new_bufs, old_bufs, type,
+      compat_mode, db_name, table_name, "utf8mb4", arg_count, &ap);
+  va_end(ap);
+  return br;
+}
+
+void destroy_miner_br(ObLogMinerBR *&br) {
+  ICDCRecord *cdc_br = br->get_br();
+  IDBMeta *db_meta = cdc_br->getDBMeta();
+  ITableMeta *tbl_meta = cdc_br->getTableMeta();
+  delete static_cast<libobcdc::ObLogBR*>(cdc_br->getUserData());
+  DRCMessageFactory::destroy(db_meta);
+  DRCMessageFactory::destroy(tbl_meta);
+  DRCMessageFactory::destroy(cdc_br);
+  delete br;
+  br = nullptr;
+}
+
+ObLogMinerRecord *build_logminer_record(ObIAllocator &alloc,
+                  lib::Worker::CompatMode	compat_mode,
+                  uint64_t tenant_id,
+                  int64_t orig_cluster_id,
+                  const char *tenant_name,
+                  const char *database_name,
+                  const char *tbl_name,
+                  int64_t trans_id,
+                  const char* const * pks,
+                  const int64_t pk_cnt,
+                  const char* const * uks,
+                  const int64_t uk_cnt,
+                  const char* row_unique_id,
+                  RecordType record_type,
+                  int64_t commit_ts_us,
+                  const char * redo_stmt,
+                  const char * undo_stmt)
+{
+  void *ptr = ob_malloc(sizeof(ObLogMinerRecord), "testminer_rec");
+  ObLogMinerRecord *rec = new (ptr) (ObLogMinerRecord);
+  rec->set_allocator(&alloc);
+
+  rec->compat_mode_ = compat_mode;
+  rec->tenant_id_ = tenant_id;
+  rec->orig_cluster_id_ = orig_cluster_id;
+  rec->tenant_name_.assign(tenant_name);
+  rec->database_name_.assign(database_name);
+  rec->table_name_.assign(tbl_name);
+  rec->trans_id_ = trans_id;
+  for (int i = 0; i < pk_cnt; i++) {
+    rec->primary_keys_.push_back(pks[i]);
+  }
+  for (int i = 0; i < uk_cnt; i++) {
+    rec->unique_keys_.push_back(uks[i]);
+  }
+  rec->row_unique_id_ = row_unique_id;
+  rec->record_type_ = record_type;
+  rec->commit_scn_.convert_from_ts(commit_ts_us);
+  rec->redo_stmt_.append(redo_stmt);
+  rec->undo_stmt_.append(undo_stmt);
+
+  return rec;
+}
+
+void destroy_miner_record(ObLogMinerRecord *&rec)
+{
+  rec->redo_stmt_.reset();
+  rec->undo_stmt_.reset();
+  ob_free(rec);
+}
+}
+}

unittest/logminer/ob_log_miner_test_utils.h

@@ -0,0 +1,68 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_LOG_MINER_UNITTEST_UTILS_H_
+#define OCEANBASE_LOG_MINER_UNITTEST_UTILS_H_
+
+#include "ob_log_miner_br.h"
+#include "rpc/obmysql/ob_mysql_global.h"
+#include "gtest/gtest.h"
+#define private public
+#include "ob_log_miner_record.h"
+#undef private
+namespace oceanbase
+{
+namespace oblogminer
+{
+ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs,
+              binlogBuf *old_bufs,
+              RecordType type,
+              lib::Worker::CompatMode compat_mode,
+              const char *db_name,
+              const char *table_name,
+              const int arg_count, ...);
+
+ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs,
+              binlogBuf *old_bufs,
+              RecordType type,
+              lib::Worker::CompatMode compat_mode,
+              const char *db_name,
+              const char *table_name,
+              const char *encoding,
+              const int arg_count, ...);
+
+void destroy_miner_br(ObLogMinerBR *&br);
+
+ObLogMinerRecord *build_logminer_record(ObIAllocator &alloc,
+                  lib::Worker::CompatMode	compat_mode,
+                  uint64_t tenant_id,
+                  int64_t orig_cluster_id,
+                  const char *tenant_name,
+                  const char *database_name,
+                  const char *tbl_name,
+                  int64_t trans_id,
+                  const char* const * pks,
+                  const int64_t pk_cnt,
+                  const char* const * uks,
+                  const int64_t uk_cnt,
+                  const char* row_unique_id,
+                  RecordType record_type,
+                  int64_t commit_ts_us,
+                  const char * redo_stmt,
+                  const char * undo_stmt);
+
+void destroy_miner_record(ObLogMinerRecord *&rec);
+
+}
+}
+
+#endif

unittest/logminer/test_ob_log_miner_analyzer_checkpoint.cpp

@@ -0,0 +1,70 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_miner_analyzer_checkpoint.h"
+#include "lib/oblog/ob_log.h"
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_log_miner_analyzer_ckpt, SerializeFunc)
+{
+  ObLogMinerCheckpoint ckpt, ckpt1;
+  int64_t pos = 0;
+  const char *buf1 = "PROGRESS=1\nCUR_FILE_ID=3\nMAX_FILE_ID=2\n";
+  char tmp_buf[1000];
+  EXPECT_EQ(OB_SUCCESS, ckpt.deserialize(buf1, strlen(buf1), pos));
+  EXPECT_EQ(ckpt.progress_, 1);
+  EXPECT_EQ(ckpt.cur_file_id_, 3);
+  EXPECT_EQ(ckpt.max_file_id_, 2);
+  pos = 0;
+  EXPECT_EQ(OB_SUCCESS, ckpt.serialize(tmp_buf, 1000, pos));
+  tmp_buf[pos+1] = '\0';
+  EXPECT_STREQ(buf1, tmp_buf);
+  pos = 0;
+  const char *buf2 = "PROGRESS=a\nCUR_FILE_ID=3\nMAX_FILE_ID=2\n";
+  EXPECT_EQ(OB_INVALID_DATA, ckpt.deserialize(buf2, strlen(buf2), pos));
+  pos = 0;
+  const char *buf3 = "PROGRESS=1\nCUR_FILE_ID=3MAX_FILE_ID=2\n";
+  EXPECT_EQ(OB_INVALID_DATA, ckpt.deserialize(buf3, strlen(buf3), pos));
+  pos = 0;
+
+  for (int i = 0; i < 10000; i++) {
+    int64_t pos1 = 0;
+    pos = 0;
+    ckpt.cur_file_id_ = rand();
+    ckpt.max_file_id_ = rand();
+    ckpt.progress_ = rand();
+    EXPECT_EQ(OB_SUCCESS, ckpt.serialize(tmp_buf, 1000, pos));
+    EXPECT_EQ(pos, ckpt.get_serialize_size());
+    EXPECT_EQ(OB_SUCCESS, ckpt1.deserialize(tmp_buf, pos, pos1));
+    EXPECT_EQ(ckpt, ckpt1);
+  }
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_analyzer_checkpoint.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_analyzer_checkpoint.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_br_filter.cpp

@@ -0,0 +1,520 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "gtest/gtest.h"
+#include "lib/oblog/ob_log.h"
+#include "ob_log_miner_br_filter.h"
+#include "ob_log_miner_filter_condition.h"
+#include "ob_log_miner_br.h"
+#include "ob_log_miner_test_utils.h"
+
+
+using namespace oceanbase;
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(ob_log_miner_br_filter, ColumnBRFilterPlugin)
+{
+  ObArenaAllocator arena_alloc("FilterTest");
+  ColumnBRFilterPlugin col_filter(&arena_alloc);
+  ObLogMinerBR *br = nullptr;
+  bool need_filter = false;
+  const int buf_cnt = 10;
+  binlogBuf *new_buf = static_cast<binlogBuf*>(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt));
+  binlogBuf *old_buf = static_cast<binlogBuf*>(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt));
+  for (int i = 0; i < 10; i++) {
+    new_buf[i].buf = static_cast<char*>(arena_alloc.alloc(1024));
+    new_buf[i].buf_size = 1024;
+    new_buf[i].buf_used_size = 0;
+    old_buf[i].buf = static_cast<char*>(arena_alloc.alloc(1024));
+    old_buf[i].buf_size = 1024;
+    old_buf[i].buf_used_size = 0;
+  }
+  EXPECT_EQ(OB_SUCCESS, col_filter.init("[]"));
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "val1", "val2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"table_name\":\"aaa\", \
+  \"table_name\":\"aaa\"}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"tenant_name\":\"aaa\", \
+  \"table_name\":\"aaa\"}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"aaa\", \
+  \"database_name\":\"aaa\"}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"t1.aaa\", \
+  \"table_name\":\"aaa\"}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"t1.aaa\", \
+  \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": 3.1415926}]}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_SUCCESS, col_filter.init("[{\"database_name\":\"t1.aaa\", \
+  \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"}]}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"t1.aaa\", \
+  \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_SUCCESS, col_filter.init("[{\"database_name\":\"t1.aaa\", \
+  \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"}, {\"col3\":null}]}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[\"table_name\":\"aaa\", "
+      "\"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"},{\"col3\":null}]}]"));
+  col_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[\"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"},{\"col3\":null}]}]"));
+  col_filter.destroy();
+
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("db1.t1|db2.t2|db3.t*"));
+  col_filter.destroy();
+
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("db1.*|db2"));
+  col_filter.destroy();
+
+  EXPECT_EQ(OB_SUCCESS, col_filter.init("["
+    "{\"database_name\":\"db1\",\"table_name\":\"table1\","
+      "\"column_cond\":[{\"col1\":\"val1\"},{\"col2\":\"val2\"}]"
+    "}"
+  "]"));
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "val1", "val2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col2", "val3", "val2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col2", "val3", "val3",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), "col1", nullptr, "val1",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db2", "table1", 8, "col2", "val3", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col1", "val1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table2", 8, "col2", "val3", "val3",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col1", nullptr, "val1", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col2", "val3", "val3",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col3", nullptr, "val4", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+  br = build_logminer_br(new_buf, old_buf, EDDL, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table2", 0);
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+  col_filter.destroy();
+
+  EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("db*.t1|*_prod.*_history|db3.t*"));
+  col_filter.destroy();
+
+  EXPECT_EQ(OB_SUCCESS, col_filter.init(
+    "[{\"database_name\":\"db1\", \"table_name\":\"table1\","
+    "\"column_cond\":[{\"col1\":\"val1\", \"col2\":\"val2\"}]}]"));
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db2", "non_spec_tbl", 3, "non_spec_col", "non_spec_val", nullptr);
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db0", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db3", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db3", "tmp_tbl", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val2", "val2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val1", "val3", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val1", "val1",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val1", "val3", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val10", "val100",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val2", "val2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val1", "val2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val1", "val2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", nullptr, "val1",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", nullptr, "val2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 12, "col1", "val10", "val1",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val20", "val2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col3", "val3", "val30", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+  col_filter.destroy();
+
+  // null value filter
+  EXPECT_EQ(OB_SUCCESS, col_filter.init(
+    "[{\"database_name\":\"db1\", \"table_name\":\"table1\","
+    "\"column_cond\":[{\"col1\":null}]}]"));
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db2", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE,lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "non_spec_val", "xx",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", nullptr, nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", nullptr, nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", nullptr, "xx",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+  col_filter.destroy();
+
+  // multi column filter with null
+  EXPECT_EQ(OB_SUCCESS, col_filter.init(
+    "[{\"database_name\":\"db1\", \"table_name\":\"table1\","
+    "\"column_cond\":[{\"col1\":\"val1\", \"col2\":\"val2\"}, {\"col3\":null}]}]"));
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db2", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT,lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col3", nullptr, nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val1", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val2", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "val1", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+  col_filter.destroy();
+
+
+  // multi column filter
+  EXPECT_EQ(OB_SUCCESS, col_filter.init(
+    "[{\"database_name\":\"db1\", \"table_name\":\"table1\","
+    "\"column_cond\":[{\"col1\":\"val1\", \"col2\":\"val2\"}, {\"col3\":\"val3\", \"col4\":\"val4\"}]}]"));
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db2", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "val1", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val1", nullptr,
+       static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+       "col2", "val2", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 8, "col1", "val1", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col3", "val3", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+  col_filter.destroy();
+}
+
+TEST(ob_log_miner_br_filter, OperationBRFilterPlugin)
+{
+  OperationBRFilterPlugin op_filter;
+  ObArenaAllocator arena_alloc("OpFilterTest");
+  const int64_t buf_cnt = 10;
+  ObLogMinerBR *br = nullptr;
+  bool need_filter = false;
+  binlogBuf *new_buf = static_cast<binlogBuf*>(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt));
+  binlogBuf *old_buf = static_cast<binlogBuf*>(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt));
+  for (int i = 0; i < 10; i++) {
+    new_buf[i].buf = static_cast<char*>(arena_alloc.alloc(1024));
+    new_buf[i].buf_size = 1024;
+    new_buf[i].buf_used_size = 0;
+    old_buf[i].buf = static_cast<char*>(arena_alloc.alloc(1024));
+    old_buf[i].buf_size = 1024;
+    old_buf[i].buf_used_size = 0;
+  }
+  EXPECT_EQ(OB_INVALID_ARGUMENT, op_filter.init("aaa|bbb"));
+  op_filter.destroy();
+  EXPECT_EQ(OB_SUCCESS, op_filter.init("Insert"));
+  op_filter.destroy();
+  EXPECT_EQ(OB_INVALID_ARGUMENT, op_filter.init("lnsert"));
+  op_filter.destroy();
+
+  EXPECT_EQ(OB_ERR_UNEXPECTED, op_filter.init(nullptr));
+  op_filter.destroy();
+
+
+  EXPECT_EQ(OB_SUCCESS, op_filter.init("insert"));
+
+  br = build_logminer_br(new_buf, old_buf, EBEGIN, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, HEARTBEAT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  op_filter.destroy();
+
+  EXPECT_EQ(OB_SUCCESS, op_filter.init("insert|update"));
+
+  br = build_logminer_br(new_buf, old_buf, HEARTBEAT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  op_filter.destroy();
+
+  EXPECT_EQ(OB_SUCCESS, op_filter.init("insert|update|delete"));
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", "val10", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 12, "col1", "val10", "val1",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col2", "val20", "val2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "col3", "val3", "val30", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 4, "col1", nullptr, "val1",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EBEGIN, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, ECOMMIT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, HEARTBEAT, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EBEGIN, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(false, need_filter);
+  destroy_miner_br(br);
+
+  br = build_logminer_br(new_buf, old_buf, EREPLACE, lib::Worker::CompatMode::MYSQL,
+      "tenant1.db1", "table1", 0);
+  EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter));
+  EXPECT_EQ(true, need_filter);
+  destroy_miner_br(br);
+
+  op_filter.destroy();
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_filter.log");
+  ObLogger &logger = ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_filter.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_file_index.cpp

@@ -0,0 +1,160 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define private public
+#include "ob_log_miner_file_index.h"
+#undef private
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+static bool operator==(const FileIndex &lhs, const FileIndex &rhs)
+{
+  bool bret = true;
+
+  if (lhs.index_file_len_ != rhs.index_file_len_) {
+    bret = false;
+  } else if (lhs.index_array_.count() != rhs.index_array_.count()) {
+    bret = false;
+  } else {
+    for (int64_t i = 0; bret && i < lhs.index_array_.count(); i++) {
+      const FileIndexItem *lhs_item = lhs.index_array_.at(i);
+      const FileIndexItem *rhs_item = rhs.index_array_.at(i);
+      if (*lhs_item == *rhs_item) {
+      } else {
+        bret = false;
+      }
+    }
+  }
+
+  return bret;
+}
+
+TEST(test_ob_log_miner_file_index, FileIndexItem)
+{
+  FileIndexItem item, item1;
+  const int64_t buf_len = 300;
+  char buf[buf_len];
+  int64_t pos = 0;
+  const char *buf1 = "1:2:3\n";
+  EXPECT_EQ(OB_SUCCESS, item.deserialize(buf1, strlen(buf1), pos));
+  EXPECT_EQ(item.file_id_, 1);
+  EXPECT_EQ(item.range_.min_commit_ts_, 2);
+  EXPECT_EQ(item.range_.max_commit_ts_, 3);
+  pos = 0;
+  const char *buf2 = "1:2:3";
+  EXPECT_EQ(OB_SIZE_OVERFLOW, item.deserialize(buf2, strlen(buf2), pos));
+  pos = 0;
+  const char *buf3 = "1:a:3\n";
+  EXPECT_EQ(OB_INVALID_DATA, item.deserialize(buf3, strlen(buf3), pos));
+  pos = 0;
+  const char *buf4 = "1|2|3\n";
+  EXPECT_EQ(OB_INVALID_DATA, item.deserialize(buf4, strlen(buf4), pos));
+  pos = 0;
+
+  for (int i = 0; i < 10000; i++) {
+    item1.file_id_ = rand();
+    item1.range_.min_commit_ts_ = rand();
+    item1.range_.max_commit_ts_ = rand();
+    EXPECT_EQ(OB_SUCCESS, item1.serialize(buf, buf_len, pos));
+    EXPECT_EQ(item1.get_serialize_size(), pos);
+    pos = 0;
+    EXPECT_EQ(OB_SUCCESS, item.deserialize(buf, buf_len, pos));
+    EXPECT_EQ(item, item1);
+    pos = 0;
+  }
+}
+
+TEST(test_ob_log_miner_file_index, FileIndex)
+{
+  FileIndex index2, index1,index;
+  FileIndexItem *item = nullptr;
+  const int64_t buf_size = 1000;
+  char buf[buf_size];
+  char buffer[buf_size];
+  int64_t pos = 0;
+  int64_t position = 0;
+  const char *buf1 = "0:2:3\n1:3:3\n2:4:7\n";
+  EXPECT_EQ(OB_SUCCESS, index.deserialize(buf1, strlen(buf1), pos));
+  EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(0,2,3));
+  EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(1,3,3));
+  EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(2,4,7));
+  EXPECT_EQ(index, index1);
+  EXPECT_EQ(OB_ERROR_OUT_OF_RANGE, index.get_index_item(-1, item));
+  EXPECT_EQ(OB_SUCCESS, index.get_index_item(0, item));
+  EXPECT_EQ(OB_SUCCESS, index.get_index_item(1, item));
+  EXPECT_EQ(OB_SUCCESS, index.get_index_item(2, item));
+  EXPECT_EQ(OB_ERROR_OUT_OF_RANGE, index.get_index_item(3, item));
+  pos = 0;
+  index.reset();
+  index1.reset();
+  index2.reset();
+  const char *buf2 = "1:2:3\n4:5:6\n3:4:5\n";
+  EXPECT_EQ(OB_SUCCESS, index.deserialize(buf2, strlen(buf2), pos));
+  EXPECT_EQ(OB_ERR_UNEXPECTED, index.get_index_item(0, item));
+  pos = 0;
+  index.reset();
+  index1.reset();
+  index2.reset();
+  const char *buf3 = "0:1:2\n1:2:3\n";
+  EXPECT_EQ(OB_SUCCESS, index.deserialize(buf3, strlen(buf3), pos));
+  EXPECT_EQ(strlen(buf3), index.get_index_file_len());
+  pos = 0;
+  index.reset();
+  index1.reset();
+  index2.reset();
+  for(int i = 0; i < 1000; i++) {
+    index.reset();
+    index1.reset();
+    int item_cnt = abs(rand()) % 10 + 1;
+    for (int j = 0; j < item_cnt; j++) {
+      FileIndexItem item;
+      item.reset(j, rand(), rand());
+      EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(item));
+      EXPECT_EQ(OB_SUCCESS, item.serialize(buffer, buf_size, position));
+    }
+    EXPECT_EQ(position, index1.get_serialize_size());
+    EXPECT_EQ(position, index1.get_index_file_len());
+    int64_t data_len = position;
+    position = 0;
+    EXPECT_EQ(OB_SUCCESS, index2.deserialize(buffer, data_len, position));
+    EXPECT_EQ(index1, index2);
+    EXPECT_EQ(OB_SUCCESS, index1.serialize(buf, buf_size, pos));
+    data_len = pos;
+    pos = 0;
+    EXPECT_EQ(OB_SUCCESS, index.deserialize(buf, data_len, pos));
+    EXPECT_EQ(index, index1);
+    pos = 0;
+    position = 0;
+    index.reset();
+    index1.reset();
+    index2.reset();
+  }
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_file_index.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_file_index.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_file_manager.cpp

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define private public
+#include "ob_log_miner_file_manager.h"
+#undef private
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_ob_log_miner_file_manager, GenFileHeader)
+{
+  ObLogMinerFileManager manager;
+  const int64_t path_size = 100;
+  char cwd_buf[path_size];
+  ObArenaAllocator alloc;
+  char *header = nullptr;
+  int64_t data_len = 0;
+  EXPECT_NE(nullptr, getcwd(cwd_buf, path_size));
+  char path[OB_MAX_URI_LENGTH];
+  char rm_dir_command[OB_MAX_URI_LENGTH];
+  EXPECT_EQ(OB_SUCCESS, databuff_printf(path, sizeof(path), "file://%s/logminer_output", cwd_buf));
+  EXPECT_EQ(OB_SUCCESS, databuff_printf(rm_dir_command, sizeof(rm_dir_command), "rm -rf %s/logminer_output", cwd_buf));
+  EXPECT_EQ(OB_SUCCESS,
+      manager.init(path, RecordFileFormat::CSV, ObLogMinerFileManager::FileMgrMode::ANALYZE));
+  EXPECT_EQ(OB_SUCCESS, manager.generate_data_file_header_(alloc, header, data_len));
+  EXPECT_STREQ(header, "TENANT_ID,TRANS_ID,PRIMARY_KEY,TENANT_NAME,DATABASE_NAME,"
+  "TABLE_NAME,OPERATION,OPERATION_CODE,COMMIT_SCN,COMMIT_TIMESTAMP,SQL_REDO,SQL_UNDO,ORG_CLUSTER_ID\n");
+
+  system(rm_dir_command);
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_file_manager.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_file_manager.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_file_meta.cpp

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_miner_file_meta.h"
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_log_miner_file_meta, SerializeFunc)
+{
+  ObLogMinerFileMeta file_meta, file_meta1;
+  const int64_t buf_len = 300;
+  char buf[buf_len];
+  int64_t pos = 0;
+  const char *buf1 = "MIN_COMMIT_TS=1\nMAX_COMMIT_TS=2\nDATA_LEN=3\n";
+  EXPECT_EQ(OB_SUCCESS, file_meta.deserialize(buf1, strlen(buf1), pos));
+  EXPECT_EQ(pos, strlen(buf1));
+  EXPECT_EQ(file_meta.range_.min_commit_ts_, 1);
+  EXPECT_EQ(file_meta.range_.max_commit_ts_, 2);
+  EXPECT_EQ(file_meta.data_length_, 3);
+  pos = 0;
+  const char *buf2 = "MIN_COMMIT_TS=1a\nMAX_COMMIT_TS=2\nDATA_LEN=3\n";
+  EXPECT_EQ(OB_INVALID_DATA, file_meta.deserialize(buf2, strlen(buf2), pos));
+  pos = 0;
+  const char *buf3 = "MIN_COMMIT_TS=11\nMAX_COMMIT_TS=2\nDATA_LEN=3";
+  EXPECT_EQ(OB_SIZE_OVERFLOW, file_meta.deserialize(buf3, strlen(buf3), pos));
+  pos = 0;
+
+  for (int i = 0; i < 10000; i++) {
+    file_meta1.data_length_ = rand();
+    file_meta1.range_.min_commit_ts_ = rand();
+    file_meta1.range_.max_commit_ts_ = rand();
+    EXPECT_EQ(OB_SUCCESS, file_meta1.serialize(buf, buf_len, pos));
+    EXPECT_EQ(pos, file_meta1.get_serialize_size());
+    pos = 0;
+    EXPECT_EQ(OB_SUCCESS, file_meta.deserialize(buf, buf_len, pos));
+    EXPECT_EQ(file_meta, file_meta1);
+    pos = 0;
+  }
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_file_meta.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_file_meta.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_progress_range.cpp

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_miner_progress_range.h"
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_log_miner_progress_range, SerializeFunc)
+{
+  ObLogMinerProgressRange range, range1;
+  const int64_t buf_len = 100;
+  char buf[buf_len];
+  int64_t pos = 0;
+  const char *buf1 = "MIN_COMMIT_TS=1\nMAX_COMMIT_TS=2\n";
+  EXPECT_EQ(OB_SUCCESS, range.deserialize(buf1, strlen(buf1), pos));
+  EXPECT_EQ(range.min_commit_ts_, 1);
+  EXPECT_EQ(range.max_commit_ts_, 2);
+  pos = 0;
+  const char *buf2 = "MIN_COMMIT_TS=aaa\nMAX_COMMIT_TS=2\n";
+  EXPECT_EQ(OB_INVALID_DATA, range.deserialize(buf2, strlen(buf2), pos));
+  pos = 0;
+  const char *buf3 = "MIN_COMMIT_TS:1\nMAX_COMMIT_TS:2\n";
+  EXPECT_EQ(OB_INVALID_DATA, range.deserialize(buf3, strlen(buf3), pos));
+  pos = 0;
+  const char *buf4 = "MIN_COMMIT_TS=1MAX_COMMIT_TS=2\n";
+  EXPECT_EQ(OB_INVALID_DATA, range.deserialize(buf4, strlen(buf4), pos));
+  pos = 0;
+  for (int i = 0; i < 10000; i++) {
+    range1.min_commit_ts_ = rand();
+    range1.max_commit_ts_ = rand();
+    EXPECT_EQ(range1.serialize(buf, buf_len, pos), OB_SUCCESS);
+    EXPECT_EQ(pos, range1.get_serialize_size());
+    pos = 0;
+    EXPECT_EQ(range.deserialize(buf, buf_len, pos), OB_SUCCESS);
+    EXPECT_EQ(range.get_serialize_size(), pos);
+    EXPECT_EQ(range, range1);
+    pos = 0;
+  }
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_progress_range.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_progress_range.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_record.cpp

@@ -0,0 +1,795 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "lib/oblog/ob_log.h"
+#include "ob_log_miner_test_utils.h"
+#include "gtest/gtest.h"
+#define private public
+#include "ob_log_miner_record.h"
+#undef private
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_ob_log_miner_record, InitObLogMinerRecord)
+{
+  ObArenaAllocator allocator("TestLogMnrRec");
+  ObLogMinerBR *br = nullptr;
+  ObLogMinerRecord record;
+  const int buf_cnt = 10;
+  binlogBuf *new_buf = static_cast<binlogBuf*>(allocator.alloc(sizeof(binlogBuf) * buf_cnt));
+  binlogBuf *old_buf = static_cast<binlogBuf*>(allocator.alloc(sizeof(binlogBuf) * buf_cnt));
+  for (int i = 0; i < 10; i++) {
+    new_buf[i].buf = static_cast<char*>(allocator.alloc(1024));
+    new_buf[i].buf_size = 1024;
+    new_buf[i].buf_used_size = 0;
+    old_buf[i].buf = static_cast<char*>(allocator.alloc(1024));
+    old_buf[i].buf_size = 1024;
+    old_buf[i].buf_used_size = 0;
+  }
+  record.set_allocator(&allocator);
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "t1.db1", "tbl1", 4, "id", "1", nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("t1", record.tenant_name_.ptr());
+  EXPECT_STREQ("db1", record.database_name_.ptr());
+  EXPECT_STREQ("tbl1", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db1`.`tbl1` (`id`) VALUES ('1');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db1`.`tbl1` WHERE `id`='1' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", 8, "id", "1", "2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "name", "aaa", "bbb", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `id`='1', `name`='aaa' WHERE `id`='2' AND `name`='bbb' LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `id`='2', `name`='bbb' WHERE `id`='1' AND `name`='aaa' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", 8, "id", nullptr , "2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "name", nullptr, "bbb", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='bbb' LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', 'bbb');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDDL, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "", 4, "ddl_stmt_str", "CREATE TABLE T1(ID INT PRIMARY KEY);" , nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("CREATE TABLE T1(ID INT PRIMARY KEY);", record.redo_stmt_.ptr());
+  EXPECT_STREQ("/* NO SQL_UNDO GENERATED */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "gbk", 8,
+      "id", nullptr , "2",static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "name", nullptr, "\xB0\xC2\xD0\xC7\xB1\xB4\xCB\xB9",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF' LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', '\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf16", 4, "name", nullptr, "\x59\x65\x66\x1F\x8D\x1D\x65\xAF",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `name`='\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF' LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`) VALUES ('\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "latin1", 8, "id", nullptr , "2",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "name", nullptr, "\x63\x69\xe0\x6f",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='\x63\x69\xc3\xa0\x6f' LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', '\x63\x69\xc3\xa0\x6f');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "gb18030", 8,
+      "id", nullptr , "2",static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING),
+      "name", nullptr, "\x95\xcb\x95\xcb\xc6\xe4\xd2\xf5\xa3\xac\xc6\xe4\xec\x59\xf2\xb3\xf2\xb3",
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='\xe6\x9b\x80\xe6\x9b\x80\xe5\x85\xb6\xe9\x98\xb4\xef\xbc\x8c\xe5\x85\xb6\xe9\x9d\x81\xe8\x99\xba\xe8\x99\xba' LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', '\xe6\x9b\x80\xe6\x9b\x80\xe5\x85\xb6\xe9\x98\xb4\xef\xbc\x8c\xe5\x85\xb6\xe9\x9d\x81\xe8\x99\xba\xe8\x99\xba');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "gb18030", 4,
+      "id", nullptr , "2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"id\"=2 and ROWNUM=1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES (2);", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "gb18030", 4,
+      "id", nullptr , "2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"id\"='2' and ROWNUM=1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES ('2');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "gb18030", 4,
+      "id", "2" , nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES ('2');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"id\"='2' and ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "gb18030", 4,
+      "id", "2" , nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_BLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES ('2');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE 0=DBMS_LOB.COMPARE(\"id\", '2') and ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 4,
+      "name", "ABCD" , nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_BLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"name\") VALUES (HEXTORAW('41424344'));", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE 0=DBMS_LOB.COMPARE(\"name\", HEXTORAW('41424344')) and ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "binary", 4,
+      "name", "ABCD" , nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_BLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`) VALUES (UNHEX('41424344'));", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `name`=UNHEX('41424344') LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8, "name", "ABCD" , nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING),
+      "col2", "XXX", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING));
+  br->get_br()->getTableMeta()->setUKs("name");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`) VALUES ('ABCD', 'XXX');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `name`='ABCD' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8, "name", "ABCD" , nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING),
+      "col2", "XXX", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING));
+  br->get_br()->getTableMeta()->setPKs("col2");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`) VALUES ('ABCD', 'XXX');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`='XXX' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12, "name", "ABCD" , nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING),
+      "col2", "XXX", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING),
+      "col3", "val3", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING));
+  br->get_br()->getTableMeta()->setPKs("col2,col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`, `col3`) VALUES ('ABCD', 'XXX', 'val3');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`='XXX' AND `col3`='val3' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12, "name", "ABCD" , nullptr,
+      static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING),
+      "col2", "XXX", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING),
+      "col3", "val3", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING));
+  br->get_br()->getTableMeta()->setUKs("col2,col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`, `col3`) VALUES ('ABCD', 'XXX', 'val3');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`='XXX' AND `col3`='val3' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG),
+      "col2", "2", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG),
+      "col3", "3", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col2,col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES (1, 2, 3);", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`=2 AND `col3`=3 LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_BIT),
+      "col2", "1836032", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_BIT));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES (b'1', b'111000000010000000000');", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1`=b'1' AND `col2`=b'111000000010000000000' LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+}
+
+TEST(test_ob_log_miner_record, LobTypeInMySqlMode)
+{
+  ObArenaAllocator allocator("TestLogMnrRec");
+  ObLogMinerBR *br = nullptr;
+  ObLogMinerRecord record;
+  const int buf_cnt = 10;
+  binlogBuf *new_buf = static_cast<binlogBuf*>(allocator.alloc(sizeof(binlogBuf) * buf_cnt));
+  binlogBuf *old_buf = static_cast<binlogBuf*>(allocator.alloc(sizeof(binlogBuf) * buf_cnt));
+  for (int i = 0; i < 10; i++) {
+    new_buf[i].buf = static_cast<char*>(allocator.alloc(1024));
+    new_buf[i].buf_size = 1024;
+    new_buf[i].buf_used_size = 0;
+    old_buf[i].buf = static_cast<char*>(allocator.alloc(1024));
+    old_buf[i].buf_size = 1024;
+    old_buf[i].buf_used_size = 0;
+  }
+  record.set_allocator(&allocator);
+
+
+  // mysql multimode
+  // insert without key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", "{\"key\": \"value\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES ('{\"key\": \"value\"}', ST_GeomFromText('POINT(0 1)'));",record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1`=cast('{\"key\": \"value\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // insert with key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", "{\"key\": \"value\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES ('{\"key\": \"value\"}', ST_GeomFromText('POINT(0 1)'), 1);", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=1 LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value insert without key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES (NULL, ST_GeomFromText('POINT(0 1)'));",record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1` IS NULL AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value insert with key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", "{\"key\": \"value\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES ('{\"key\": \"value\"}', NULL, 1);", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=1 LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // update without key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)') WHERE `col1`=cast('{\"key\": \"old\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"old\"}', `col2`=ST_GeomFromText('POINT(2 3)') WHERE `col1`=cast('{\"key\": \"new\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // update with key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "1", "2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)'), `col3`=1 WHERE `col3`=2 LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"old\"}', `col2`=ST_GeomFromText('POINT(2 3)'), `col3`=2 WHERE `col3`=1 LIMIT 1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value update without key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", "{\"key\": \"new\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)') WHERE `col1` IS NULL AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`=NULL, `col2`=ST_GeomFromText('POINT(2 3)') WHERE `col1`=cast('{\"key\": \"new\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value update with key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", "{\"key\": \"new\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "POINT(0 1)", "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)'), `col3`=1 WHERE `col3` IS NULL LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`=NULL, `col2`=ST_GeomFromText('POINT(2 3)'), `col3`=NULL WHERE `col3`=1 LIMIT 1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // delete without key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", nullptr, "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1`=cast('{\"key\": \"old\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES ('{\"key\": \"old\"}', ST_GeomFromText('POINT(2 3)'));", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // delete with key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", nullptr, "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, "2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=2 LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES ('{\"key\": \"old\"}', ST_GeomFromText('POINT(2 3)'), 2);", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value delete without key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 8,
+      "col1", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1` IS NULL AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES (NULL, ST_GeomFromText('POINT(2 3)'));/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value delete with key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL,
+      "tenant2.db2", "tbl2", "utf8mb4", 12,
+      "col1", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "POINT(2 3)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, "1", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col3");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=1 LIMIT 1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES (NULL, ST_GeomFromText('POINT(2 3)'), 1);/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+}
+
+TEST(test_ob_log_miner_record, LobTypeInOracleMode)
+{
+  ObArenaAllocator allocator("TestLogMnrRec");
+  ObLogMinerBR *br = nullptr;
+  ObLogMinerRecord record;
+  const int buf_cnt = 10;
+  binlogBuf *new_buf = static_cast<binlogBuf*>(allocator.alloc(sizeof(binlogBuf) * buf_cnt));
+  binlogBuf *old_buf = static_cast<binlogBuf*>(allocator.alloc(sizeof(binlogBuf) * buf_cnt));
+  for (int i = 0; i < 10; i++) {
+    new_buf[i].buf = static_cast<char*>(allocator.alloc(1024));
+    new_buf[i].buf_size = 1024;
+    new_buf[i].buf_used_size = 0;
+    old_buf[i].buf = static_cast<char*>(allocator.alloc(1024));
+    old_buf[i].buf_size = 1024;
+    old_buf[i].buf_used_size = 0;
+  }
+  record.set_allocator(&allocator);
+  // multimode
+  // insert without key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 16,
+      "col1", "{\"key\": \"value\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), '<a>abc</a>', 'AABB1122');",record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE JSON_EQUAL(\"col1\", '{\"key\": \"value\"}') AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='<a>abc</a>' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // insert with key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 20,
+      "col1", "{\"key\": \"value\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB),
+      "col5", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col5");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), '<a>abc</a>', 'AABB1122', 1);", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=1 and ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value insert without key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 16,
+      "col1", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES (NULL, SDO_GEOMETRY('POINT(0 1)', NULL), '<a>abc</a>', 'AABB1122');",record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col1\" IS NULL AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='<a>abc</a>' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value insert with key
+  br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 20,
+      "col1", "{\"key\": \"value\"}", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB),
+      "col5", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col5");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', NULL, '<a>abc</a>', 'AABB1122', 1);", record.redo_stmt_.ptr());
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=1 and ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // update without key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 16,
+      "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "SRID=NULL;POINT(0 1)", "SRID=NULL;POINT(2 1)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", "<a>cba</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", "1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL),"
+      " \"col3\"='<a>abc</a>', \"col4\"='AABB1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"old\"}') AND "
+      "\"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL) AND \"col3\"='<a>cba</a>' AND 0=DBMS_LOB.COMPARE(\"col4\", '1122')"
+      " AND ROWNUM=1;/* POTENTIALLY INACCURATE */", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL),"
+      " \"col3\"='<a>cba</a>', \"col4\"='1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"new\"}') AND "
+      "\"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='<a>abc</a>' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122')"
+      " AND ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // update with key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 20,
+      "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "SRID=NULL;POINT(0 1)", "SRID=NULL;POINT(2 1)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", "<a>cba</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", "1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB),
+      "col5", "1", "2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col5");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL), \"col3\"='<a>abc</a>', \"col4\"='AABB1122', \"col5\"=1 WHERE \"col5\"=2 AND ROWNUM=1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL), \"col3\"='<a>cba</a>', \"col4\"='1122', \"col5\"=2 WHERE \"col5\"=1 AND ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value update without key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 16,
+      "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", "1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL),"
+      " \"col3\"=NULL, \"col4\"='AABB1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"old\"}') AND \"col2\" IS NULL AND "
+      "\"col3\" IS NULL AND 0=DBMS_LOB.COMPARE(\"col4\", '1122') AND ROWNUM=1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=NULL, \"col3\"=NULL,"
+      " \"col4\"='1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"new\"}') AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL)"
+      " AND \"col3\" IS NULL AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') AND ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value update with key
+  br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 20,
+      "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", "SRID=NULL;POINT(0 1)", "SRID=NULL;POINT(2 1)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", "<a>abc</a>", "<a>cba</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", "AABB1122", "1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB),
+      "col5", "1", nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col5");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL), \"col3\"='<a>abc</a>', \"col4\"='AABB1122', \"col5\"=1 WHERE \"col5\" IS NULL AND ROWNUM=1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL), \"col3\"='<a>cba</a>', \"col4\"='1122', \"col5\"=NULL WHERE \"col5\"=1 AND ROWNUM=1;", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // delete without key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 16,
+      "col1", nullptr, "{\"key\": \"value\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "SRID=NULL;POINT(0 1)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, "<a>abc</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", nullptr, "AABB1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE JSON_EQUAL(\"col1\", '{\"key\": \"value\"}') AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='<a>abc</a>' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), '<a>abc</a>', 'AABB1122');", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // delete with key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 20,
+      "col1", nullptr, "{\"key\": \"value\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "SRID=NULL;POINT(0 1)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, "<a>abc</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", nullptr, "AABB1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB),
+      "col5", nullptr, "2", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col5");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=2 and ROWNUM=1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), '<a>abc</a>', 'AABB1122', 2);", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value delete without key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 16,
+      "col1", nullptr, "{\"key\": \"value\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, nullptr, static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, "<a>abc</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", nullptr, "AABB1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB));
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE JSON_EQUAL(\"col1\", '{\"key\": \"value\"}') AND \"col2\" IS NULL AND \"col3\"='<a>abc</a>' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES ('{\"key\": \"value\"}', NULL, '<a>abc</a>', 'AABB1122');/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+
+  // null value delete with key
+  br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE,
+      "tenant2.db2", "tbl2", "utf8mb4", 20,
+      "col1", nullptr, "{\"key\": \"value\"}", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON),
+      "col2", nullptr, "SRID=NULL;POINT(0 1)", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY),
+      "col3", nullptr, "<a>abc</a>", drcmsg_field_types::DRCMSG_TYPE_ORA_XML,
+      "col4", nullptr, "AABB1122", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB),
+      "col5", nullptr, "1", static_cast<int>(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG));
+  br->get_br()->getTableMeta()->setUKs("col5");
+  EXPECT_EQ(OB_SUCCESS, record.init(*br));
+  EXPECT_STREQ("tenant2", record.tenant_name_.ptr());
+  EXPECT_STREQ("db2", record.database_name_.ptr());
+  EXPECT_STREQ("tbl2", record.table_name_.ptr());
+  EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br));
+  EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=1 and ROWNUM=1;", record.redo_stmt_.ptr());
+  EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), '<a>abc</a>', 'AABB1122', 1);", record.undo_stmt_.ptr());
+  destroy_miner_br(br);
+  record.destroy();
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_record.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_record.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_record_converter.cpp

@@ -0,0 +1,313 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_log_miner_timezone_getter.h"
+#include "ob_log_miner_test_utils.h"
+#include "gtest/gtest.h"
+
+#define private public
+#include "ob_log_miner_record_converter.h"
+#undef private
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_ob_log_miner_record_converter, CsvConverterWriteType)
+{
+  ObLogMinerRecordCsvConverter converter;
+  ObConcurrentFIFOAllocator alloc;
+  ObStringBuffer str_buf(&alloc);
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  EXPECT_EQ(OB_SUCCESS, converter.write_csv_string_escape_("'aaaa\"\"bbbbb'", str_buf));
+  EXPECT_STREQ("\"'aaaa\"\"\"\"bbbbb'\"", str_buf.ptr());
+  str_buf.reset();
+}
+
+TEST(test_ob_log_miner_record_converter, JsonConverterWriteType)
+{
+  ObLogMinerRecordJsonConverter converter;
+  ObConcurrentFIFOAllocator alloc;
+  ObStringBuffer str_buf(&alloc);
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  EXPECT_EQ(OB_SUCCESS, converter.write_json_key_("aaa", str_buf));
+  EXPECT_STREQ("\"aaa\":", str_buf.ptr());
+  str_buf.reset();
+  EXPECT_EQ(OB_SUCCESS, converter.write_json_string_escape_("'aaaa\"\"bbbbb'", str_buf));
+  EXPECT_STREQ("\"'aaaa\\\"\\\"bbbbb'\"", str_buf.ptr());
+  str_buf.reset();
+  EXPECT_EQ(OB_SUCCESS, converter.write_json_string_escape_("\n\b\t", str_buf));
+  EXPECT_STREQ("\"\\n\\b\\t\"", str_buf.ptr());
+  str_buf.reset();
+}
+
+TEST(test_ob_log_miner_record_converter, CsvConverterWriteRecord)
+{
+  ObLogMinerRecordCsvConverter converter;
+  ObConcurrentFIFOAllocator alloc;
+  EXPECT_EQ(OB_SUCCESS, LOGMINER_TZ.set_timezone("+8:00"));
+  bool is_written = false;
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  ObStringBuffer str_buf(&alloc);
+  ObLogMinerRecord *rec = nullptr;
+  const char *pkarr1[] = {"aaa", "bbb"};
+  const char *ukarr1[] = {"ccc"};
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222,
+        "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');",
+        "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';");
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "1002,345,\"aaa/bbb\",\"test_tenant\",\"test\",\"sbtest1\",\"INSERT\",1,1645539742222222000,\"2022-02-22 22:22:22.222222\","
+    "\"INSERT INTO \"\"test\"\".\"\"sbtest1\"\"(\"\"aaa\"\",\"\"bbb\"\",\"\"ccc\"\") VALUES('1','2','3');\","
+    "\"DELETE FROM \"\"test\"\".\"\"sbtest1\"\" WHERE \"\"aaa\"\"='1' and \"\"bbb\"\"='2' and \"\"ccc\"\"='3';\","
+    "1\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EUPDATE, 0,
+        "UPDATE \"test\".\"sbtest1\" SET \"aaa\" = '1', \"bbb\" = '2', \"ccc\" = '3' WHERE "
+        "\"aaa\" = '4' AND \"bbb\" = '5' and \"ccc\" = '6' LIMIT 1;",
+        "UPDATE \"test\".\"sbtest1\" SET \"aaa\" = '4', \"bbb\" = '5', \"ccc\" = '6' WHERE "
+        "\"aaa\" = '1' AND \"bbb\" = '2' and \"ccc\" = '3' LIMIT 1;");
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "1002,345,\"aaa/bbb\",\"test_tenant\",\"test\",\"sbtest1\",\"UPDATE\",2,0,\"1970-01-01 08:00:00.000000\","
+    "\"UPDATE \"\"test\"\".\"\"sbtest1\"\" SET \"\"aaa\"\" = '1', \"\"bbb\"\" = '2', \"\"ccc\"\" = '3' WHERE "
+    "\"\"aaa\"\" = '4' AND \"\"bbb\"\" = '5' and \"\"ccc\"\" = '6' LIMIT 1;\","
+    "\"UPDATE \"\"test\"\".\"\"sbtest1\"\" SET \"\"aaa\"\" = '4', \"\"bbb\"\" = '5', \"\"ccc\"\" = '6' WHERE "
+    "\"\"aaa\"\" = '1' AND \"\"bbb\"\" = '2' and \"\"ccc\"\" = '3' LIMIT 1;\","
+    "1\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "", 345, nullptr,0,
+        nullptr, 0, nullptr, EDDL, 4611686018427387,
+        "CREATE TABLE t1(id INT, name TEXT);",
+        nullptr);
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "1002,345,\"\",\"test_tenant\",\"test\",\"\",\"DDL\",4,4611686018427387000,\"2116-02-21 07:53:38.427387\","
+    "\"CREATE TABLE t1(id INT, name TEXT);\","
+    "\"\","
+    "1\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EDELETE, 0,
+        "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';",
+        "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');"
+        );
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "1002,345,\"aaa/bbb\",\"test_tenant\",\"test\",\"sbtest1\",\"DELETE\",3,0,\"1970-01-01 08:00:00.000000\","
+    "\"DELETE FROM \"\"test\"\".\"\"sbtest1\"\" WHERE \"\"aaa\"\"='1' and \"\"bbb\"\"='2' and \"\"ccc\"\"='3';\","
+    "\"INSERT INTO \"\"test\"\".\"\"sbtest1\"\"(\"\"aaa\"\",\"\"bbb\"\",\"\"ccc\"\") VALUES('1','2','3');\","
+    "1\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+}
+
+TEST(test_ob_log_miner_record_converter, RedoSqlConverterWriteRecord)
+{
+  ObLogMinerRecordRedoSqlConverter converter;
+  ObConcurrentFIFOAllocator alloc;
+  bool is_written = false;
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  ObStringBuffer str_buf(&alloc);
+  ObLogMinerRecord *rec = nullptr;
+  const char *pkarr1[] = {"aaa", "bbb"};
+  const char *ukarr1[] = {"ccc"};
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222,
+        "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');",
+        "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';");
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "", 345, nullptr,0,
+        nullptr, 0, nullptr, EDDL, 4611686018427387,
+        "CREATE TABLE t1(id INT, name TEXT);",
+        nullptr);
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "CREATE TABLE t1(id INT, name TEXT);\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "", 345, nullptr,0,
+        nullptr, 0, nullptr, EBEGIN, 4611686018427387,
+        nullptr,
+        nullptr);
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(false, is_written);
+  destroy_miner_record(rec);
+}
+
+TEST(test_ob_log_miner_record_converter, UndoSqlConverterWriteRecord)
+{
+  ObLogMinerRecordUndoSqlConverter converter;
+  ObConcurrentFIFOAllocator alloc;
+  bool is_written = false;
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  ObStringBuffer str_buf(&alloc);
+  ObLogMinerRecord *rec = nullptr;
+  const char *pkarr1[] = {"aaa", "bbb"};
+  const char *ukarr1[] = {"ccc"};
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222,
+        "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');",
+        "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';");
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "", 345, nullptr, 0,
+        nullptr, 0, nullptr, EDDL, 4611686018427387,
+        "CREATE TABLE t1(id INT, name TEXT);",
+        nullptr);
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(false, is_written);
+  str_buf.reset();
+  destroy_miner_record(rec);
+
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "", 345, nullptr,0,
+        nullptr, 0, nullptr, EBEGIN, 4611686018427387,
+        nullptr,
+        nullptr);
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(false, is_written);
+  destroy_miner_record(rec);
+}
+
+TEST(test_ob_log_miner_record_converter, JsonConverterWriteRecord)
+{
+  ObLogMinerRecordJsonConverter converter;
+  ObConcurrentFIFOAllocator alloc;
+  EXPECT_EQ(OB_SUCCESS, LOGMINER_TZ.set_timezone("+8:00"));
+  bool is_written = false;
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  ObStringBuffer str_buf(&alloc);
+  ObLogMinerRecord *rec = nullptr;
+  const char *pkarr1[] = {"aaa", "bbb"};
+  const char *ukarr1[] = {"ccc"};
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222,
+        "INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');",
+        "DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;");
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"aaa/bbb\",\"TENANT_NAME\":\"test_tenant\",\"DATABASE_NAME\":\"test\","
+    "\"TABLE_NAME\":\"sbtest1\",\"OPERATION\":\"INSERT\",\"OPERATION_CODE\":1,\"COMMIT_SCN\":1645539742222222000,"
+    "\"COMMIT_TIMESTAMP\":\"2022-02-22 22:22:22.222222\","
+    "\"SQL_REDO\":\"INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');\","
+    "\"SQL_UNDO\":\"DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;\","
+    "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest2", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EUPDATE, 0,
+        "UPDATE `test`.`sbtest2` SET `aaa`='44', `bbb`='55', `ccc`='66' WHERE `aaa`='11' AND `bbb`='22' AND `ccc`='33' LIMIT 1;",
+        "UPDATE `test`.`sbtest2` SET `aaa`='11', `bbb`='22', `ccc`='33' WHERE `aaa`='44' AND `bbb`='55' AND `ccc`='66' LIMIT 1;");
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"aaa/bbb\",\"TENANT_NAME\":\"test_tenant\","
+    "\"DATABASE_NAME\":\"test\",\"TABLE_NAME\":\"sbtest2\",\"OPERATION\":\"UPDATE\",\"OPERATION_CODE\":2,"
+    "\"COMMIT_SCN\":0,\"COMMIT_TIMESTAMP\":\"1970-01-01 08:00:00.000000\","
+    "\"SQL_REDO\":\"UPDATE `test`.`sbtest2` SET `aaa`='44', `bbb`='55', `ccc`='66' WHERE `aaa`='11' AND `bbb`='22' AND `ccc`='33' LIMIT 1;\","
+    "\"SQL_UNDO\":\"UPDATE `test`.`sbtest2` SET `aaa`='11', `bbb`='22', `ccc`='33' WHERE `aaa`='44' AND `bbb`='55' AND `ccc`='66' LIMIT 1;\","
+    "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "", 345, nullptr,0,
+        nullptr, 0, nullptr, EDDL, 4611686018427387,
+        "CREATE TABLE `sbtest2` (\n  `aaa` varchar(100) NOT NULL,\n  `bbb` varchar(100) NOT NULL,\n  `ccc` varchar(100) DEFAULT NULL\n);",
+        nullptr);
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"\",\"TENANT_NAME\":\"test_tenant\",\"DATABASE_NAME\":\"test\","
+    "\"TABLE_NAME\":\"\",\"OPERATION\":\"DDL\",\"OPERATION_CODE\":4,\"COMMIT_SCN\":4611686018427387000,\"COMMIT_TIMESTAMP\":\"2116-02-21 07:53:38.427387\","
+    "\"SQL_REDO\":\"CREATE TABLE `sbtest2` (\\n  `aaa` varchar(100) NOT NULL,\\n  `bbb` varchar(100) NOT NULL,\\n  `ccc` varchar(100) DEFAULT NULL\\n);\",\"SQL_UNDO\":\"\","
+    "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+  rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL,
+        1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*),
+        ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EDELETE, 0,
+        "DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;",
+        "INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');"
+        );
+  is_written = false;
+  EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written));
+  EXPECT_EQ(true, is_written);
+  EXPECT_STREQ(
+    "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"aaa/bbb\",\"TENANT_NAME\":\"test_tenant\",\"DATABASE_NAME\":\"test\","
+    "\"TABLE_NAME\":\"sbtest1\",\"OPERATION\":\"DELETE\",\"OPERATION_CODE\":3,\"COMMIT_SCN\":0,"
+    "\"COMMIT_TIMESTAMP\":\"1970-01-01 08:00:00.000000\","
+    "\"SQL_REDO\":\"DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;\","
+    "\"SQL_UNDO\":\"INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');\","
+    "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr());
+  str_buf.reset();
+  destroy_miner_record(rec);
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_record_converter.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_record_converter.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}

unittest/logminer/test_ob_log_miner_utils.cpp

@@ -0,0 +1,310 @@
+/**
+ * Copyright (c) 2023 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "lib/allocator/ob_concurrent_fifo_allocator.h"     //ObConcurrentFIFOAllocator
+#include "lib/allocator/page_arena.h"
+#include "ob_log_miner_utils.h"
+#include "gtest/gtest.h"
+
+namespace oceanbase
+{
+namespace oblogminer
+{
+
+TEST(test_ob_log_miner_utils, ObLogMinerDeepCopyCstring)
+{
+  const char *archive_dest = "archive_dest";
+  char *val = nullptr;
+  ObArenaAllocator alloc;
+  EXPECT_EQ(OB_SUCCESS, deep_copy_cstring(alloc, archive_dest, val));
+  EXPECT_STREQ(archive_dest, val);
+  EXPECT_EQ(OB_ERR_UNEXPECTED, deep_copy_cstring(alloc, archive_dest, val));
+  val = nullptr;
+  EXPECT_EQ(OB_SUCCESS, deep_copy_cstring(alloc, nullptr, val));
+  EXPECT_EQ(nullptr, val);
+  EXPECT_EQ(OB_SUCCESS, deep_copy_cstring(alloc, "", val));
+  EXPECT_STREQ("", val);
+}
+TEST(test_ob_log_miner_utils, ObLogMinerUtilParseLineString)
+{
+  const char *key_str = "PROGRESS";
+  const char *buf = "PROGRESS=abcde\n";
+  int64_t pos = 0;
+  ObString value;
+  ObArenaAllocator alloc;
+  char *cstr = nullptr;
+  EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, value));
+  EXPECT_EQ(value, ObString("abcde"));
+  EXPECT_EQ(pos, strlen(buf));
+  pos = 0;
+  key_str = "CUR_FILE_ID";
+  EXPECT_EQ(OB_INVALID_DATA, parse_line(key_str, buf, strlen(buf), pos, value));
+  buf = "CUR_FILE_ID=001\n";
+  pos = 0;
+  EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, value));
+  EXPECT_EQ(ObString("001"), value);
+  EXPECT_EQ(pos, strlen(buf));
+  pos = 0;
+  key_str = "kkkk";
+  buf = "kkkk=llll\n";
+  EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, alloc, cstr));
+  EXPECT_STREQ("llll", cstr);
+  const char *charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwlyz_-.";
+  const int64_t charset_len = strlen(charset);
+  char data_buf[100];
+  char key[30];
+  char val[30];
+  for (int i = 0; i < 10000; i++) {
+    int64_t kv_len = abs(rand()) % (sizeof(key)-1) + 1;
+    pos = 0;
+    for (int j = 0; j < kv_len; j++) {
+      int k = rand() % charset_len;
+      int v = rand() % charset_len;
+      key[j] = charset[k];
+      val[j] = charset[v];
+    }
+    key[kv_len] = '\0';
+    val[kv_len] = '\0';
+    snprintf(data_buf, 100, "%s=%s\n", key, val);
+    EXPECT_EQ(OB_SUCCESS, parse_line(key, data_buf, sizeof(data_buf), pos, value));
+    EXPECT_EQ(ObString(val), value);
+    pos = 0;
+    EXPECT_EQ(OB_SUCCESS, parse_line(key, data_buf, sizeof(data_buf), pos, alloc, cstr));
+    EXPECT_STREQ(val, cstr);
+  }
+}
+
+TEST(test_ob_log_miner_utils, ObLogMinerUtilsStrToLL)
+{
+  int64_t num = 0;
+  EXPECT_EQ(OB_SUCCESS, logminer_str2ll("111222", num));
+  EXPECT_EQ(111222, num);
+  EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll("9223372036854775808", num));
+  EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll("-9223372036854775809", num));
+  EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll(nullptr, num));
+  EXPECT_EQ(OB_SUCCESS, logminer_str2ll("12345aaa", num));
+  EXPECT_EQ(12345, num);
+  const char *timestr = "1688473439\n";
+  char *end_ptr = nullptr;
+  EXPECT_EQ(OB_SUCCESS, logminer_str2ll(timestr, end_ptr, num));
+  EXPECT_EQ(1688473439, num);
+  EXPECT_EQ(end_ptr, timestr + strlen(timestr)-1);
+  timestr = "aaaaa";
+  EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll(timestr, end_ptr, num));
+  EXPECT_EQ(end_ptr, timestr);
+}
+
+TEST(test_ob_log_miner_utils, ObLogMinerUtilsParseLine)
+{
+  const char *key_str = "PROGRESS";
+  const char *buf = "PROGRESS=1234567\n";
+  int64_t pos = 0;
+  int64_t data = 0;
+  EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, data));
+  EXPECT_EQ(data, 1234567);
+  EXPECT_EQ(pos, strlen(buf));
+  pos = 0;
+  key_str = "CUR_FILE_ID";
+  EXPECT_EQ(OB_INVALID_DATA, parse_line(key_str, buf, strlen(buf), pos, data));
+  buf = "CUR_FILE_ID=001\n";
+  pos = 0;
+  EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, data));
+  EXPECT_EQ(1, data);
+  EXPECT_EQ(pos, strlen(buf));
+  buf = "CUR_FILE_ID:1\n";
+  pos = 0;
+  EXPECT_EQ(OB_INVALID_DATA, parse_line(key_str, buf, strlen(buf), pos, data));
+  buf = "CUR_FILE_ID=1";
+  pos = 0;
+  EXPECT_EQ(OB_SIZE_OVERFLOW, parse_line(key_str, buf, strlen(buf), pos, data));
+  const char *charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwlyz_-.";
+  const int64_t charset_len = strlen(charset);
+  char data_buf[100];
+  char key[30];
+  for (int i = 0; i < 10000; i++) {
+    int64_t key_len = abs(rand()) % (sizeof(key)-1) + 1;
+    pos = 0;
+    for (int j = 0; j < key_len; j++) {
+      int k = rand() % charset_len;
+      key[j] = charset[k];
+    }
+    key[key_len] = '\0';
+    int64_t val = rand();
+    snprintf(data_buf, 100, "%s=%ld\n", key, val);
+    EXPECT_EQ(OB_SUCCESS, parse_line(key, data_buf, sizeof(data_buf), pos, data));
+    EXPECT_EQ(val, data);
+  }
+}
+
+TEST(test_ob_log_miner_utils, ParseDigit)
+{
+
+  int64_t pos = 0;
+  int64_t data = 0;
+  const char *buf = "12345667";
+  EXPECT_EQ(OB_SUCCESS, parse_int(buf, strlen(buf), pos, data));
+  EXPECT_EQ(data, 12345667);
+  EXPECT_EQ(pos, strlen(buf));
+  pos = 0; data = 0;
+  const char buf0[] = {'1','2','3','4','5'};
+  EXPECT_EQ(OB_SUCCESS, parse_int(buf0, sizeof(buf0), pos, data));
+  EXPECT_EQ(data, 12345);
+  EXPECT_EQ(pos, sizeof(buf0));
+  pos = 0; data = 0;
+  const char *buf2 = "  +1234567aaa";
+  EXPECT_EQ(OB_SUCCESS, parse_int(buf2, strlen(buf2), pos, data));
+  EXPECT_EQ(data, 1234567);
+  EXPECT_EQ(pos, 10);
+  pos = 0; data = 0;
+  const char *buf3 = "  -1234567aaa";
+  EXPECT_EQ(OB_SUCCESS, parse_int(buf3, strlen(buf3), pos, data));
+  EXPECT_EQ(data, -1234567);
+  EXPECT_EQ(pos, 10);
+  pos = 0; data = 0;
+  const char *buf4 = "9223372036854775807";
+  EXPECT_EQ(OB_SUCCESS, parse_int(buf4, strlen(buf4), pos, data));
+  EXPECT_EQ(data, 9223372036854775807LL);
+  EXPECT_EQ(pos, strlen(buf4));
+  pos = 0; data = 0;
+  const char *buf5 = "-9223372036854775808";
+  EXPECT_EQ(OB_SUCCESS, parse_int(buf5, strlen(buf5), pos, data));
+  EXPECT_EQ(data, -9223372036854775807LL - 1);
+  EXPECT_EQ(pos, strlen(buf5));
+  pos = 0; data = 0;
+  const char *buf6 = "9223372036854775808";
+  EXPECT_EQ(OB_SIZE_OVERFLOW, parse_int(buf6, strlen(buf6), pos, data));
+  pos = 0; data = 0;
+  const char *buf7 = "-9223372036854775809";
+  EXPECT_EQ(OB_SIZE_OVERFLOW, parse_int(buf7, strlen(buf7), pos, data));
+  pos = 0; data = 0;
+  const char *buf_invalid = "+-111";
+  EXPECT_EQ(OB_INVALID_DATA, parse_int(buf_invalid, strlen(buf_invalid), pos, data));
+  pos = 0; data = 0;
+  buf_invalid = "  +  111";
+  EXPECT_EQ(OB_INVALID_DATA, parse_int(buf_invalid, strlen(buf_invalid), pos, data));
+  pos = 0; data = 0;
+  buf_invalid = "  a111";
+  EXPECT_EQ(OB_INVALID_DATA, parse_int(buf_invalid, strlen(buf_invalid), pos, data));
+  pos = 0; data = 0;
+  char buf_tmp[100];
+  for (int i = 0; i < 10000; i++) {
+    pos = 0;
+    int64_t data1 = rand();
+    EXPECT_EQ(OB_SUCCESS, databuff_printf(buf_tmp, 100, pos, "CUR_FILE_ID="));
+    int64_t tmp_pos = pos;
+    EXPECT_EQ(OB_SUCCESS, databuff_printf(buf_tmp, 100, pos, "%ld\n", data1));
+    EXPECT_EQ(OB_SUCCESS, parse_int(buf_tmp, 100, tmp_pos, data));
+    EXPECT_EQ(data, data1);
+  }
+}
+
+TEST(test_ob_log_miner_utils, ExpectToken)
+{
+  const char *buf = "MIN_COMMIT_TS=1";
+  int64_t pos = 0;
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "MIN_COMMIT_TS"));
+  EXPECT_EQ(pos, 13);
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "="));
+  EXPECT_EQ(pos, 14);
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "1"));
+  EXPECT_EQ(pos, 15);
+  pos = 0;
+
+  buf = "abcdefghxxxx";
+  EXPECT_EQ(OB_SIZE_OVERFLOW, expect_token(buf, strlen(buf), pos, "abcdefghxxxxx"));
+  pos = 0;
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "abcdefgh"));
+  EXPECT_EQ(pos, 8);
+  EXPECT_EQ(OB_SIZE_OVERFLOW, expect_token(buf, strlen(buf), pos, "xxxxx"));
+  pos = 8;
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "xxxx"));
+  pos = 0;
+
+  buf = "CUR_FILE_ID=1\n";
+  EXPECT_EQ(OB_INVALID_DATA, expect_token(buf, strlen(buf), pos, "MAX_FILE_ID"));
+  pos = 0;
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "CUR_FILE_ID"));
+  EXPECT_EQ(pos, 11);
+  EXPECT_EQ(OB_INVALID_DATA, expect_token(buf, strlen(buf), pos, "1"));
+  pos = 11;
+  EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "=1\n"));
+  EXPECT_EQ(pos, 14);
+  pos = 0;
+}
+
+TEST(test_ob_log_miner_utils, ConvertUintToBit)
+{
+  uint64_t val = 0;
+  ObArenaAllocator tmp_alloc;
+  ObStringBuffer bit_str(&tmp_alloc);
+  EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str));
+  EXPECT_STREQ("0", bit_str.ptr());
+  bit_str.reset();
+
+  val = 1;
+  EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str));
+  EXPECT_STREQ("1", bit_str.ptr());
+  bit_str.reset();
+
+  val = 6;
+  EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str));
+  EXPECT_STREQ("110", bit_str.ptr());
+  bit_str.reset();
+
+  val = 1836032;
+  EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str));
+  EXPECT_STREQ("111000000010000000000", bit_str.ptr());
+  bit_str.reset();
+
+  val = 183848324234;
+  EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str));
+  EXPECT_STREQ("10101011001110001101101100110010001010", bit_str.ptr());
+  bit_str.reset();
+}
+
+TEST(test_ob_log_miner_utils, ObLogMinerUtilsWriteRecord)
+{
+  ObConcurrentFIFOAllocator alloc;
+  ObStringBuffer str_buf(&alloc);
+  KeyArray key_arr;
+  EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13));
+  EXPECT_EQ(key_arr.push_back("aaa"), OB_SUCCESS);
+  EXPECT_EQ(key_arr.push_back("bbb"), OB_SUCCESS);
+  EXPECT_EQ(key_arr.push_back("ccc"), OB_SUCCESS);
+  EXPECT_EQ(OB_SUCCESS, write_keys(key_arr, str_buf));
+  EXPECT_STREQ("\"aaa/bbb/ccc\"", str_buf.ptr());
+  str_buf.reset();
+  EXPECT_EQ(OB_SUCCESS, write_signed_number(-12345, str_buf));
+  EXPECT_STREQ("-12345", str_buf.ptr());
+  str_buf.reset();
+  EXPECT_EQ(OB_SUCCESS, write_unsigned_number(1111, str_buf));
+  EXPECT_STREQ("1111", str_buf.ptr());
+  str_buf.reset();
+  EXPECT_EQ(OB_SUCCESS, write_string_no_escape("aaaaabbbb'", str_buf));
+  EXPECT_STREQ("\"aaaaabbbb'\"", str_buf.ptr());
+  str_buf.reset();
+}
+
+}
+}
+
+int main(int argc, char **argv)
+{
+  // testing::FLAGS_gtest_filter = "DO_NOT_RUN";
+  system("rm -f test_ob_log_miner_utils.log");
+  oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger();
+  logger.set_file_name("test_ob_log_miner_utils.log", true, false);
+  logger.set_log_level("DEBUG");
+  logger.set_enable_async_log(false);
+  testing::InitGoogleTest(&argc,argv);
+  return RUN_ALL_TESTS();
+}