From ed28f133d6320b0a02161ba6e35d7e28675c518f Mon Sep 17 00:00:00 2001 From: nautaa <870284156@qq.com> Date: Fri, 31 May 2024 04:59:38 +0000 Subject: [PATCH] [FEAT MERGE] oblogminer[patch from 4.2.3] Co-authored-by: qiuyg3 --- cmake/Pack.cmake | 2 + deps/oblib/src/lib/oblog/ob_log.cpp | 5 + deps/oblib/src/lib/oblog/ob_log.h | 2 + deps/oblib/src/lib/oblog/ob_log_module.h | 5 + deps/oblib/src/lib/oblog/ob_log_module.ipp | 1 + src/logservice/CMakeLists.txt | 1 + src/logservice/libobcdc/src/ob_log_config.h | 2 +- .../libobcdc/src/ob_log_instance.cpp | 4 +- src/logservice/logminer/CMakeLists.txt | 44 + src/logservice/logminer/ob_log_miner.cpp | 144 ++ src/logservice/logminer/ob_log_miner.h | 93 ++ .../logminer/ob_log_miner_analysis_writer.cpp | 150 ++ .../logminer/ob_log_miner_analysis_writer.h | 78 + .../logminer/ob_log_miner_analyze_schema.h | 31 + .../logminer/ob_log_miner_analyzer.cpp | 239 ++++ .../logminer/ob_log_miner_analyzer.h | 65 + .../ob_log_miner_analyzer_checkpoint.cpp | 68 + .../ob_log_miner_analyzer_checkpoint.h | 70 + src/logservice/logminer/ob_log_miner_args.cpp | 920 ++++++++++++ src/logservice/logminer/ob_log_miner_args.h | 201 +++ .../logminer/ob_log_miner_batch_record.cpp | 219 +++ .../logminer/ob_log_miner_batch_record.h | 200 +++ .../ob_log_miner_batch_record_writer.cpp | 383 +++++ .../ob_log_miner_batch_record_writer.h | 131 ++ src/logservice/logminer/ob_log_miner_br.cpp | 79 + src/logservice/logminer/ob_log_miner_br.h | 136 ++ .../logminer/ob_log_miner_br_converter.cpp | 202 +++ .../logminer/ob_log_miner_br_converter.h | 79 + .../logminer/ob_log_miner_br_filter.cpp | 469 ++++++ .../logminer/ob_log_miner_br_filter.h | 141 ++ .../logminer/ob_log_miner_br_producer.cpp | 330 +++++ .../logminer/ob_log_miner_br_producer.h | 83 ++ .../logminer/ob_log_miner_data_manager.cpp | 300 ++++ .../logminer/ob_log_miner_data_manager.h | 144 ++ .../logminer/ob_log_miner_error_handler.h | 30 + .../logminer/ob_log_miner_file_index.cpp | 208 +++ .../logminer/ob_log_miner_file_index.h | 103 ++ .../logminer/ob_log_miner_file_manager.cpp | 661 +++++++++ .../logminer/ob_log_miner_file_manager.h | 151 ++ .../logminer/ob_log_miner_file_meta.cpp | 62 + .../logminer/ob_log_miner_file_meta.h | 59 + .../ob_log_miner_filter_condition.cpp | 427 ++++++ .../logminer/ob_log_miner_filter_condition.h | 188 +++ .../ob_log_miner_flashback_reader.cpp | 11 + .../logminer/ob_log_miner_flashback_reader.h | 29 + .../ob_log_miner_flashback_writer.cpp | 11 + .../logminer/ob_log_miner_flashback_writer.h | 34 + .../logminer/ob_log_miner_logger.cpp | 104 ++ src/logservice/logminer/ob_log_miner_logger.h | 51 + src/logservice/logminer/ob_log_miner_main.cpp | 55 + src/logservice/logminer/ob_log_miner_mode.cpp | 98 ++ src/logservice/logminer/ob_log_miner_mode.h | 41 + .../logminer/ob_log_miner_progress_range.cpp | 62 + .../logminer/ob_log_miner_progress_range.h | 68 + .../logminer/ob_log_miner_record.cpp | 1267 +++++++++++++++++ src/logservice/logminer/ob_log_miner_record.h | 279 ++++ .../ob_log_miner_record_aggregator.cpp | 255 ++++ .../logminer/ob_log_miner_record_aggregator.h | 88 ++ .../ob_log_miner_record_converter.cpp | 445 ++++++ .../logminer/ob_log_miner_record_converter.h | 90 ++ .../ob_log_miner_record_file_format.cpp | 73 + .../ob_log_miner_record_file_format.h | 38 + .../logminer/ob_log_miner_record_filter.cpp | 11 + .../logminer/ob_log_miner_record_filter.h | 36 + .../logminer/ob_log_miner_record_parser.cpp | 11 + .../logminer/ob_log_miner_record_parser.h | 34 + .../logminer/ob_log_miner_record_rewriter.cpp | 11 + .../logminer/ob_log_miner_record_rewriter.h | 36 + .../logminer/ob_log_miner_recyclable_task.h | 61 + .../ob_log_miner_resource_collector.cpp | 218 +++ .../ob_log_miner_resource_collector.h | 81 ++ .../logminer/ob_log_miner_timezone_getter.cpp | 41 + .../logminer/ob_log_miner_timezone_getter.h | 43 + .../logminer/ob_log_miner_undo_task.cpp | 11 + .../logminer/ob_log_miner_undo_task.h | 38 + .../logminer/ob_log_miner_utils.cpp | 458 ++++++ src/logservice/logminer/ob_log_miner_utils.h | 115 ++ unittest/CMakeLists.txt | 1 + unittest/logminer/CMakeLists.txt | 27 + unittest/logminer/ob_log_miner_test_utils.cpp | 186 +++ unittest/logminer/ob_log_miner_test_utils.h | 68 + .../test_ob_log_miner_analyzer_checkpoint.cpp | 70 + unittest/logminer/test_ob_log_miner_args.cpp | 1083 ++++++++++++++ .../logminer/test_ob_log_miner_br_filter.cpp | 520 +++++++ .../logminer/test_ob_log_miner_file_index.cpp | 160 +++ .../test_ob_log_miner_file_manager.cpp | 58 + .../logminer/test_ob_log_miner_file_meta.cpp | 67 + .../test_ob_log_miner_progress_range.cpp | 67 + .../logminer/test_ob_log_miner_record.cpp | 795 +++++++++++ .../test_ob_log_miner_record_converter.cpp | 313 ++++ unittest/logminer/test_ob_log_miner_utils.cpp | 310 ++++ 91 files changed, 14937 insertions(+), 2 deletions(-) create mode 100644 src/logservice/logminer/CMakeLists.txt create mode 100644 src/logservice/logminer/ob_log_miner.cpp create mode 100644 src/logservice/logminer/ob_log_miner.h create mode 100644 src/logservice/logminer/ob_log_miner_analysis_writer.cpp create mode 100644 src/logservice/logminer/ob_log_miner_analysis_writer.h create mode 100644 src/logservice/logminer/ob_log_miner_analyze_schema.h create mode 100644 src/logservice/logminer/ob_log_miner_analyzer.cpp create mode 100644 src/logservice/logminer/ob_log_miner_analyzer.h create mode 100644 src/logservice/logminer/ob_log_miner_analyzer_checkpoint.cpp create mode 100644 src/logservice/logminer/ob_log_miner_analyzer_checkpoint.h create mode 100644 src/logservice/logminer/ob_log_miner_args.cpp create mode 100644 src/logservice/logminer/ob_log_miner_args.h create mode 100644 src/logservice/logminer/ob_log_miner_batch_record.cpp create mode 100644 src/logservice/logminer/ob_log_miner_batch_record.h create mode 100644 src/logservice/logminer/ob_log_miner_batch_record_writer.cpp create mode 100644 src/logservice/logminer/ob_log_miner_batch_record_writer.h create mode 100644 src/logservice/logminer/ob_log_miner_br.cpp create mode 100644 src/logservice/logminer/ob_log_miner_br.h create mode 100644 src/logservice/logminer/ob_log_miner_br_converter.cpp create mode 100644 src/logservice/logminer/ob_log_miner_br_converter.h create mode 100644 src/logservice/logminer/ob_log_miner_br_filter.cpp create mode 100644 src/logservice/logminer/ob_log_miner_br_filter.h create mode 100644 src/logservice/logminer/ob_log_miner_br_producer.cpp create mode 100644 src/logservice/logminer/ob_log_miner_br_producer.h create mode 100644 src/logservice/logminer/ob_log_miner_data_manager.cpp create mode 100644 src/logservice/logminer/ob_log_miner_data_manager.h create mode 100644 src/logservice/logminer/ob_log_miner_error_handler.h create mode 100644 src/logservice/logminer/ob_log_miner_file_index.cpp create mode 100644 src/logservice/logminer/ob_log_miner_file_index.h create mode 100644 src/logservice/logminer/ob_log_miner_file_manager.cpp create mode 100644 src/logservice/logminer/ob_log_miner_file_manager.h create mode 100644 src/logservice/logminer/ob_log_miner_file_meta.cpp create mode 100644 src/logservice/logminer/ob_log_miner_file_meta.h create mode 100644 src/logservice/logminer/ob_log_miner_filter_condition.cpp create mode 100644 src/logservice/logminer/ob_log_miner_filter_condition.h create mode 100644 src/logservice/logminer/ob_log_miner_flashback_reader.cpp create mode 100644 src/logservice/logminer/ob_log_miner_flashback_reader.h create mode 100644 src/logservice/logminer/ob_log_miner_flashback_writer.cpp create mode 100644 src/logservice/logminer/ob_log_miner_flashback_writer.h create mode 100644 src/logservice/logminer/ob_log_miner_logger.cpp create mode 100644 src/logservice/logminer/ob_log_miner_logger.h create mode 100644 src/logservice/logminer/ob_log_miner_main.cpp create mode 100644 src/logservice/logminer/ob_log_miner_mode.cpp create mode 100644 src/logservice/logminer/ob_log_miner_mode.h create mode 100644 src/logservice/logminer/ob_log_miner_progress_range.cpp create mode 100644 src/logservice/logminer/ob_log_miner_progress_range.h create mode 100644 src/logservice/logminer/ob_log_miner_record.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record.h create mode 100644 src/logservice/logminer/ob_log_miner_record_aggregator.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record_aggregator.h create mode 100644 src/logservice/logminer/ob_log_miner_record_converter.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record_converter.h create mode 100644 src/logservice/logminer/ob_log_miner_record_file_format.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record_file_format.h create mode 100644 src/logservice/logminer/ob_log_miner_record_filter.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record_filter.h create mode 100644 src/logservice/logminer/ob_log_miner_record_parser.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record_parser.h create mode 100644 src/logservice/logminer/ob_log_miner_record_rewriter.cpp create mode 100644 src/logservice/logminer/ob_log_miner_record_rewriter.h create mode 100644 src/logservice/logminer/ob_log_miner_recyclable_task.h create mode 100644 src/logservice/logminer/ob_log_miner_resource_collector.cpp create mode 100644 src/logservice/logminer/ob_log_miner_resource_collector.h create mode 100644 src/logservice/logminer/ob_log_miner_timezone_getter.cpp create mode 100644 src/logservice/logminer/ob_log_miner_timezone_getter.h create mode 100644 src/logservice/logminer/ob_log_miner_undo_task.cpp create mode 100644 src/logservice/logminer/ob_log_miner_undo_task.h create mode 100644 src/logservice/logminer/ob_log_miner_utils.cpp create mode 100644 src/logservice/logminer/ob_log_miner_utils.h create mode 100644 unittest/logminer/CMakeLists.txt create mode 100644 unittest/logminer/ob_log_miner_test_utils.cpp create mode 100644 unittest/logminer/ob_log_miner_test_utils.h create mode 100644 unittest/logminer/test_ob_log_miner_analyzer_checkpoint.cpp create mode 100644 unittest/logminer/test_ob_log_miner_args.cpp create mode 100644 unittest/logminer/test_ob_log_miner_br_filter.cpp create mode 100644 unittest/logminer/test_ob_log_miner_file_index.cpp create mode 100644 unittest/logminer/test_ob_log_miner_file_manager.cpp create mode 100644 unittest/logminer/test_ob_log_miner_file_meta.cpp create mode 100644 unittest/logminer/test_ob_log_miner_progress_range.cpp create mode 100644 unittest/logminer/test_ob_log_miner_record.cpp create mode 100644 unittest/logminer/test_ob_log_miner_record_converter.cpp create mode 100644 unittest/logminer/test_ob_log_miner_utils.cpp diff --git a/cmake/Pack.cmake b/cmake/Pack.cmake index 0930e90074..b72a0f399f 100644 --- a/cmake/Pack.cmake +++ b/cmake/Pack.cmake @@ -49,6 +49,7 @@ install(PROGRAMS tools/import_time_zone_info.py tools/import_srs_data.py ${CMAKE_BINARY_DIR}/tools/ob_admin/ob_admin + ${CMAKE_BINARY_DIR}/src/logservice/logminer/oblogminer tools/ob_admin/io_bench/bench_io.sh ${CMAKE_BINARY_DIR}/src/observer/observer DESTINATION bin @@ -369,6 +370,7 @@ if(OB_BUILD_OBADMIN) install(PROGRAMS ${CMAKE_BINARY_DIR}/tools/ob_admin/ob_admin ${CMAKE_BINARY_DIR}/tools/ob_error/src/ob_error + ${CMAKE_BINARY_DIR}/src/logservice/logminer/oblogminer DESTINATION /usr/bin COMPONENT utils ) diff --git a/deps/oblib/src/lib/oblog/ob_log.cpp b/deps/oblib/src/lib/oblog/ob_log.cpp index af0472a4d4..1c8137db1c 100644 --- a/deps/oblib/src/lib/oblog/ob_log.cpp +++ b/deps/oblib/src/lib/oblog/ob_log.cpp @@ -1923,6 +1923,11 @@ void ObLogger::issue_dba_error(const int errcode, const char *file, const int li "errcode", errcode, "file", base_file_name, "line_no", line, "info", info_str); } +bool ObLogger::is_svr_file_opened() +{ + return log_file_[FD_SVR_FILE].is_opened(); +} + } } diff --git a/deps/oblib/src/lib/oblog/ob_log.h b/deps/oblib/src/lib/oblog/ob_log.h index fe238103d1..fbc965e427 100644 --- a/deps/oblib/src/lib/oblog/ob_log.h +++ b/deps/oblib/src/lib/oblog/ob_log.h @@ -642,6 +642,8 @@ public: // issue a ERROR message for each EDIAG log right now void issue_dba_error(const int errcode, const char *file, const int line, const char *info_str); + bool is_svr_file_opened(); + private: //@brief If version <= 0, return true. //If version > 0, return version > level_version_ and if true, update level_version_. diff --git a/deps/oblib/src/lib/oblog/ob_log_module.h b/deps/oblib/src/lib/oblog/ob_log_module.h index 7cea7d77ac..f8f5da7893 100644 --- a/deps/oblib/src/lib/oblog/ob_log_module.h +++ b/deps/oblib/src/lib/oblog/ob_log_module.h @@ -75,6 +75,7 @@ DEFINE_LOG_SUB_MOD(MDS) // multi data source DEFINE_LOG_SUB_MOD(DATA_DICT) // data_dictionary module DEFINE_LOG_SUB_MOD(MVCC) // concurrency_control DEFINE_LOG_SUB_MOD(WR) // workload repository +DEFINE_LOG_SUB_MOD(LOGMINER) // logminer LOG_MOD_END(ROOT) //statement of WRS's sub_modules @@ -459,6 +460,8 @@ LOG_MOD_END(PL) #define _MVCC_LOG(level, _fmt_, args...) _OB_MOD_LOG(MVCC, level, _fmt_, ##args) #define WR_LOG(level, info_string, args...) OB_MOD_LOG(WR, level, info_string, ##args) #define _WR_LOG(level, _fmt_, args...) _OB_MOD_LOG(WR, level, _fmt_, ##args) +#define LOGMNR_LOG(level, info_string, args...) OB_MOD_LOG(LOGMINER, level, info_string, ##args) +#define _LOGMNR_LOG(level, _fmt_, args...) _OB_MOD_LOG(LOGMINER, level, _fmt_, ##args) //dfine ParMod_SubMod_LOG #define WRS_CLUSTER_LOG(level, info_string, args...) OB_SUB_MOD_LOG(WRS, CLUSTER, level, \ @@ -1153,6 +1156,8 @@ LOG_MOD_END(PL) #define _OBLOG_DISPATCHER_LOG_RET(level, errcode, args...) { int ret = errcode; _OBLOG_DISPATCHER_LOG(level, ##args); } #define OBLOG_SORTER_LOG_RET(level, errcode, args...) { int ret = errcode; OBLOG_SORTER_LOG(level, ##args); } #define _OBLOG_SORTER_LOG_RET(level, errcode, args...) { int ret = errcode; _OBLOG_SORTER_LOG(level, ##args); } +#define LOGMNR_LOG_RET(level, errcode, args...) { int ret = errcode; LOGMNR_LOG(level, ##args); } +#define _LOGMNR_LOG_RET(level, errcode, args...) { int ret = errcode; _LOGMNR_LOG(level, ##args); } #define MDS_LOG_RET(level, errcode, args...) { int ret = errcode; MDS_LOG(level, ##args); } #define _MDS_LOG_RET(level, errcode, args...) { int ret = errcode; _MDS_LOG(level, ##args); } #define DDLOG_RET(level, errcode, args...){ int ret = errcode; DDLOG(level, ##args); } diff --git a/deps/oblib/src/lib/oblog/ob_log_module.ipp b/deps/oblib/src/lib/oblog/ob_log_module.ipp index 099d8c929b..58db02d788 100644 --- a/deps/oblib/src/lib/oblog/ob_log_module.ipp +++ b/deps/oblib/src/lib/oblog/ob_log_module.ipp @@ -53,6 +53,7 @@ REG_LOG_PAR_MOD(ARCHIVE) REG_LOG_PAR_MOD(LOGTOOL) REG_LOG_PAR_MOD(DATA_DICT) REG_LOG_PAR_MOD(BALANCE) +REG_LOG_PAR_MOD(LOGMINER) //regist WRS's sub-modules REG_LOG_SUB_MOD(WRS, CLUSTER) diff --git a/src/logservice/CMakeLists.txt b/src/logservice/CMakeLists.txt index 5f30e7b58b..d3aa42e8b2 100644 --- a/src/logservice/CMakeLists.txt +++ b/src/logservice/CMakeLists.txt @@ -238,3 +238,4 @@ ob_set_subtarget(ob_logservice common_util ob_server_add_target(ob_logservice) add_subdirectory(libobcdc) +add_subdirectory(logminer) diff --git a/src/logservice/libobcdc/src/ob_log_config.h b/src/logservice/libobcdc/src/ob_log_config.h index da6b54e294..2d235d952c 100644 --- a/src/logservice/libobcdc/src/ob_log_config.h +++ b/src/logservice/libobcdc/src/ob_log_config.h @@ -603,7 +603,7 @@ public: DEF_CAP(lob_data_storage_memory_limit, OB_CLUSTER_PARAMETER, "1G", "[128M,]", "lob data storage memory limit"); T_DEF_INT_INFT(lob_data_storage_clean_interval_sec, OB_CLUSTER_PARAMETER, 5, 1, "lob_data_storage clean task nterval in seconds"); - DEF_TIME(print_mod_memory_usage_interval, OB_CLUSTER_PARAMETER, "1m", "[0s, ]", "print mod memory usage threshold"); + DEF_TIME(print_mod_memory_usage_interval, OB_CLUSTER_PARAMETER, "1m", "[0s,]", "print mod memory usage interval"); DEF_CAP(print_mod_memory_usage_threshold, OB_CLUSTER_PARAMETER, "0M", "[0M,]", "print mod memory usage threshold"); DEF_STR(print_mod_memory_usage_label, OB_CLUSTER_PARAMETER, "|", "mod label for print memmory usage"); T_DEF_INT_INFT(max_chunk_cache_size, OB_CLUSTER_PARAMETER, 0, 0, "chunkmgr max chunk size"); diff --git a/src/logservice/libobcdc/src/ob_log_instance.cpp b/src/logservice/libobcdc/src/ob_log_instance.cpp index 2c9f559ba5..e2a5c73fef 100644 --- a/src/logservice/libobcdc/src/ob_log_instance.cpp +++ b/src/logservice/libobcdc/src/ob_log_instance.cpp @@ -418,7 +418,9 @@ int ObLogInstance::init_logger_() easy_log_level = EASY_LOG_INFO; OB_LOGGER.set_max_file_size(MAX_LOG_FILE_SIZE); OB_LOGGER.set_max_file_index(max_log_file_count); - OB_LOGGER.set_file_name(log_file, disable_redirect_log_, false); + if (!OB_LOGGER.is_svr_file_opened()) { + OB_LOGGER.set_file_name(log_file, disable_redirect_log_, false); + } OB_LOGGER.set_log_level("INFO"); OB_LOGGER.disable_thread_log_level(); OB_LOGGER.set_enable_log_limit(enable_log_limit); diff --git a/src/logservice/logminer/CMakeLists.txt b/src/logservice/logminer/CMakeLists.txt new file mode 100644 index 0000000000..9513ee0dc6 --- /dev/null +++ b/src/logservice/logminer/CMakeLists.txt @@ -0,0 +1,44 @@ +ob_set_subtarget(oblogminer_obj_list common + ob_log_miner_analysis_writer.cpp + ob_log_miner_analyzer_checkpoint.cpp + ob_log_miner_analyzer.cpp + ob_log_miner_args.cpp + ob_log_miner_batch_record.cpp + ob_log_miner_batch_record_writer.cpp + ob_log_miner_br_converter.cpp + ob_log_miner_br_filter.cpp + ob_log_miner_br_producer.cpp + ob_log_miner_br.cpp + ob_log_miner_data_manager.cpp + ob_log_miner_file_index.cpp + ob_log_miner_file_manager.cpp + ob_log_miner_file_meta.cpp + ob_log_miner_filter_condition.cpp + ob_log_miner_flashback_reader.cpp + ob_log_miner_flashback_writer.cpp + ob_log_miner_logger.cpp + ob_log_miner_mode.cpp + ob_log_miner_progress_range.cpp + ob_log_miner_record_aggregator.cpp + ob_log_miner_record_converter.cpp + ob_log_miner_record_file_format.cpp + ob_log_miner_record_filter.cpp + ob_log_miner_record_parser.cpp + ob_log_miner_record_rewriter.cpp + ob_log_miner_record.cpp + ob_log_miner_resource_collector.cpp + ob_log_miner_timezone_getter.cpp + ob_log_miner_undo_task.cpp + ob_log_miner_utils.cpp + ob_log_miner.cpp +) + +add_executable(oblogminer ob_log_miner_main.cpp) +ob_add_new_object_target(oblogminer_objs oblogminer_obj_list) +ob_add_new_object_target(oblogminer_obj_dev oblogminer_obj_list) +# ob_add_new_object_target(oblogminer_main_obj oblogminer_main) +target_include_directories(oblogminer_obj_dev INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +# add_library(oblogminer_dev SHARED) +target_link_libraries(oblogminer_objs PUBLIC obcdc_static) +target_link_libraries(oblogminer_obj_dev PUBLIC obcdc) +target_link_libraries(oblogminer PRIVATE oblogminer_objs) diff --git a/src/logservice/logminer/ob_log_miner.cpp b/src/logservice/logminer/ob_log_miner.cpp new file mode 100644 index 0000000000..9e18ee43d8 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner.cpp @@ -0,0 +1,144 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner.h" +#include "lib/ob_errno.h" +#include "lib/oblog/ob_log.h" +#include "ob_log_miner_args.h" +#include "lib/allocator/ob_malloc.h" +#include "ob_log_miner_file_manager.h" +#include "ob_log_miner_utils.h" +#include "ob_log_miner_logger.h" +#include "ob_log_miner_record_converter.h" +#include "ob_log_miner_timezone_getter.h" + +namespace oceanbase { +namespace oblogminer { + +ObLogMiner *get_logminer_instance() +{ + static ObLogMiner logminer_instance; + return &logminer_instance; +} + +ObLogMiner::ObLogMiner(): + is_inited_(false), + is_stopped_(true), + mode_(LogMinerMode::UNKNOWN), + analyzer_(nullptr), + flashbacker_(nullptr), + file_manager_(nullptr) { } + +ObLogMiner::~ObLogMiner() +{ + destroy(); +} + +int ObLogMiner::init(const ObLogMinerArgs &args) +{ + int ret = common::OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_ERROR("oblogminer was inited, no need to init again", K(is_inited_)); + } else { + mode_ = args.mode_; + LOGMINER_LOGGER.set_verbose(args.verbose_); + if (OB_FAIL(LOGMINER_TZ.set_timezone(args.timezone_))) { + LOG_ERROR("oblogminer logger set timezone failed", K(ret), K(args.timezone_)); + } + if (OB_SUCC(ret)) { + if (is_analysis_mode(mode_)) { + if (OB_FAIL(init_analyzer_(args))) { + LOG_ERROR("failed to initialize analyzer for analysis mode", K(args)); + } + } else { + // TODO: flashback mode and restart mode + // if (OB_FAIL(init_component(file_manager_, + // args.flashbacker_args_.recovery_path_, RecordFileFormat::CSV, ObLogMinerFileManager::FileMgrMode::FLASHBACK))) { + // LOG_ERROR("failed to initialize file_manager for flashback mode", K(args)); + // } else if (OB_FAIL(init_component(flashbacker_, + // args.flashbacker_args_, file_manager_))) { + // LOG_ERROR("flashbacker failed to init", K(args)); + // } else if (OB_FAIL(file_manager_->write_config(args))) { + // LOG_ERROR("failed to write config", K(args)); + // } + // is_inited_ = true; + ret = OB_NOT_SUPPORTED; + LOG_ERROR("get not supported mode", K(args)); + } + } + } + if (OB_SUCC(ret)) { + is_inited_ = true; + LOG_INFO("ObLogMiner init succeed", K(args)); + LOGMINER_STDOUT("ObLogMiner init succeed\n"); + } else { + LOG_INFO("ObLogMiner init failed", K(args)); + } + return ret; +} + +int ObLogMiner::init_analyzer_(const ObLogMinerArgs &args) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(init_component(file_manager_, + args.analyzer_args_.output_dst_, args.analyzer_args_.record_format_, ObLogMinerFileManager::FileMgrMode::ANALYZE))) { + LOG_ERROR("failed to initialize file_manager for analysis mode", K(args)); + } else if (OB_FAIL(init_component(analyzer_, args.analyzer_args_, file_manager_))) { + LOG_ERROR("analyzer failed to init", K(args)); + } else if (OB_FAIL(file_manager_->write_config(args))) { + LOG_ERROR("failed to write config", K(args)); + } else if (OB_FAIL(OB_LOGGER.set_mod_log_levels(args.analyzer_args_.log_level_))) { + LOG_ERROR("failed to set mod log levels", K(args)); + LOGMINER_STDOUT("failed to set log level, please check the log level setting\n"); + } + return ret; +} +void ObLogMiner::run() { + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMiner hasn't been initiailized yet"); + } else if (is_analysis_mode(mode_)) { + if (OB_FAIL(analyzer_->start())) { + LOG_ERROR("analyzer failed to start"); + } else { + analyzer_->wait(); + } + } else if (is_flashback_mode(mode_)) { + flashbacker_->run(); + } else { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("get not supported mode", K(mode_)); + } +} + +void ObLogMiner::destroy() { + if (IS_INIT) { + if (is_analysis_mode(mode_)) { + destroy_component(analyzer_); + destroy_component(file_manager_); + } else if (is_flashback_mode(mode_)) { + destroy_component(flashbacker_); + destroy_component(file_manager_); + } + is_inited_ = false; + mode_ = LogMinerMode::UNKNOWN; + LOG_INFO("ObLogMiner destroyed"); + LOGMINER_STDOUT("ObLogMiner destroyed\n"); + } +} + +} // namespace oblogminer +} // namespace oceanbase \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner.h b/src/logservice/logminer/ob_log_miner.h new file mode 100644 index 0000000000..74ff99c3a7 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner.h @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_H_ +#define OCEANBASE_LOG_MINER_H_ + +#include +#include "ob_log_miner_mode.h" +#include "ob_log_miner_analyzer.h" + +namespace oceanbase +{ + +namespace oblogminer +{ + +class ILogMinerBRProducer; +class ILogMinerBRFilter; +class ILogMinerBRConverter; +class ILogMinerAnalysisWriter; +class ILogMinerResourceCollector; +class ILogMinerDataManager; + +class ILogMinerFlashbackReader; +class ILogMinerRecordParser; +class ILogMinerRecordFilter; +class ILogMinerFlashbackWriter; +class ILogMinerFileManager; + +class ObLogMinerArgs; +class AnalyzerArgs; +class FlashbackerArgs; + +class ObLogMinerFlashbacker { +public: + ObLogMinerFlashbacker() {} + virtual ~ObLogMinerFlashbacker() {} + int init(const FlashbackerArgs &args, ILogMinerFileManager *file_mgr) { return 0; } + void run() {} + void stop() {} + void destroy() {} + +private: + bool is_inited_; + ILogMinerFlashbackReader *reader_; + ILogMinerRecordParser *parser_; + ILogMinerRecordFilter *filter_; + ILogMinerFlashbackWriter *writer_; +}; + +class ObLogMiner { +public: + static ObLogMiner *get_logminer_instance(); + +public: + ObLogMiner(); + ~ObLogMiner(); + + int init(const ObLogMinerArgs &args); + + // LogMiner would stop automatically if some error occurs or has finished processing logs. + // There is no need to implement some interfaces like wait/stop/etc. + void run(); + void destroy(); +private: + void set_log_params_(); + int init_analyzer_(const ObLogMinerArgs &args); + +private: + bool is_inited_; + bool is_stopped_; + + LogMinerMode mode_; + + ObLogMinerAnalyzer *analyzer_; + ObLogMinerFlashbacker *flashbacker_; + ILogMinerFileManager *file_manager_; +}; + +} // oblogminer + +} // oceanbase + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_analysis_writer.cpp b/src/logservice/logminer/ob_log_miner_analysis_writer.cpp new file mode 100644 index 0000000000..20ca236121 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analysis_writer.cpp @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_analysis_writer.h" +#include "ob_log_miner_record_aggregator.h" +#include "ob_log_miner_batch_record_writer.h" +#include "ob_log_miner_logger.h" + + +namespace oceanbase +{ +namespace oblogminer +{ + +ObLogMinerAnalysisWriter::ObLogMinerAnalysisWriter(): + is_inited_(false), + is_stopped_(true), + aggregator_(nullptr), + batch_writer_(nullptr) { } + +ObLogMinerAnalysisWriter::~ObLogMinerAnalysisWriter() +{ + destroy(); +} + +int ObLogMinerAnalysisWriter::init(const int64_t start_time_us, + ILogMinerDataManager *data_manager, + ILogMinerResourceCollector *resource_collector, + ILogMinerFileManager *file_manager, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("ObLogminerAnalysisWriter has been initialized", K(is_inited_)); + } else if (OB_ISNULL(data_manager) || OB_ISNULL(resource_collector) || OB_ISNULL(file_manager)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid argument for ObLogMinerAnalysisWriter", K(data_manager), + K(resource_collector), K(file_manager)); + } else if (OB_FAIL(init_component(batch_writer_, + data_manager, resource_collector, file_manager, err_handle))) { + LOG_ERROR("failed to init ObLogMinerBatchRecordWriter", K(batch_writer_)); + } else if (OB_FAIL(init_component(aggregator_, + start_time_us, batch_writer_, data_manager, resource_collector, err_handle))) { + LOG_ERROR("failed to init ObLogMinerRecordAggregator", K(aggregator_)); + } else { + is_inited_ = true; + LOG_INFO("ObLogMinerAnalysisWriter finished to init", KP(data_manager), KP(resource_collector), + KP(file_manager), KP(err_handle)); + LOGMINER_STDOUT_V("ObLogMinerAnalysisWriter finished to init\n"); + } + + return ret; +} + +int ObLogMinerAnalysisWriter::start() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerAnalysisWriter hasn't been initialized yet", K(is_inited_)); + } else if (OB_FAIL(batch_writer_->start())) { + LOG_ERROR("batch_writer failed to start"); + } else if (OB_FAIL(aggregator_->start())) { + LOG_ERROR("aggregator failed to start"); + } else { + is_stopped_ = false; + LOG_INFO("ObLogMinerAnalysisWriter starts"); + LOGMINER_STDOUT_V("ObLogMinerAnalysisWriter starts\n"); + } + + return ret; +} + +void ObLogMinerAnalysisWriter::stop() +{ + mark_stop_flag(); + aggregator_->stop(); + batch_writer_->stop(); + LOG_INFO("ObLogMinerAnalysisWriter stopped"); +} + +void ObLogMinerAnalysisWriter::wait() +{ + aggregator_->wait(); + batch_writer_->wait(); + LOG_INFO("ObLogMinerAnalysisWriter finished to wait"); +} + +void ObLogMinerAnalysisWriter::destroy() +{ + if (IS_INIT) { + is_inited_ = false; + is_stopped_ = true; + destroy_component(aggregator_); + destroy_component(batch_writer_); + aggregator_ = nullptr; + batch_writer_ = nullptr; + LOG_INFO("ObLogMinerAnalysisWriter destroyed"); + LOGMINER_STDOUT_V("ObLogMinerAnalysisWriter destroyed\n"); + } +} + +int ObLogMinerAnalysisWriter::push(ObLogMinerRecord *record) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerAnalysisWriter hasn't been initialized yet", K(is_inited_)); + } else if (OB_FAIL(aggregator_->push(record))) { + LOG_ERROR("failed to push record into aggregator", KPC(record)); + } + + return ret; +} + +int ObLogMinerAnalysisWriter::get_total_task_count(int64_t &task_count) +{ + int ret = OB_SUCCESS; + int64_t aggregator_record_cnt = 0, batch_writer_record_cnt = 0; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerAnalysisWriter hasn't been initialized yet", K(is_inited_)); + } else if (OB_FAIL(aggregator_->get_total_record_count(aggregator_record_cnt))) { + LOG_ERROR("failed to get aggregator total record count"); + } else if (OB_FAIL(batch_writer_->get_total_record_count(batch_writer_record_cnt))) { + LOG_ERROR("failed to get batch writer record count"); + } else { + task_count = aggregator_record_cnt + batch_writer_record_cnt; + } + + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_analysis_writer.h b/src/logservice/logminer/ob_log_miner_analysis_writer.h new file mode 100644 index 0000000000..daddc351d5 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analysis_writer.h @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_ANALYSIS_WRITER_H_ +#define OCEANBASE_LOG_MINER_ANALYSIS_WRITER_H_ + +#include "lib/thread/ob_multi_fixed_queue_thread.h" +#include "ob_log_miner_error_handler.h" +namespace oceanbase +{ + +namespace oblogminer +{ + +class ObLogMinerRecord; +class ILogMinerFileManager; +class ILogMinerRecordAggregator; +class ILogMinerBatchRecordWriter; +class ILogMinerDataManager; +class ILogMinerResourceCollector; +class ILogMinerFileManager; + +class ILogMinerAnalysisWriter +{ +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int push(ObLogMinerRecord *logminer_rec) = 0; + virtual int get_total_task_count(int64_t &task_count) = 0; +}; + +class ObLogMinerAnalysisWriter: public ILogMinerAnalysisWriter +{ +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int push(ObLogMinerRecord *logminer_rec); + virtual int get_total_task_count(int64_t &task_count); + +public: + + void mark_stop_flag() { ATOMIC_STORE(&is_stopped_, true); } + bool has_stopped() const { return ATOMIC_LOAD(&is_stopped_); } +public: + ObLogMinerAnalysisWriter(); + virtual ~ObLogMinerAnalysisWriter(); + + // aggregator and batch_writer is initialized outside analysis writer + int init(const int64_t start_time_us, + ILogMinerDataManager *data_manager, + ILogMinerResourceCollector *resource_collector, + ILogMinerFileManager *file_manager, + ILogMinerErrorHandler *err_handle); +private: + bool is_inited_; + bool is_stopped_; + ILogMinerRecordAggregator *aggregator_; + ILogMinerBatchRecordWriter *batch_writer_; +}; + +} + +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_analyze_schema.h b/src/logservice/logminer/ob_log_miner_analyze_schema.h new file mode 100644 index 0000000000..98596297ac --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analyze_schema.h @@ -0,0 +1,31 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifdef MINER_SCHEMA_DEF + +MINER_SCHEMA_DEF(TENANT_ID, 0) +MINER_SCHEMA_DEF(TRANS_ID, 1) +MINER_SCHEMA_DEF(PRIMARY_KEY, 2) +// MINER_SCHEMA_DEF(ROW_UNIQUE_ID, 3) +// MINER_SCHEMA_DEF(SEQ_NO, 4) +MINER_SCHEMA_DEF(TENANT_NAME, 5) +MINER_SCHEMA_DEF(DATABASE_NAME, 6) +MINER_SCHEMA_DEF(TABLE_NAME, 7) +MINER_SCHEMA_DEF(OPERATION, 8) +MINER_SCHEMA_DEF(OPERATION_CODE, 9) +MINER_SCHEMA_DEF(COMMIT_SCN, 10) +MINER_SCHEMA_DEF(COMMIT_TIMESTAMP, 11) +MINER_SCHEMA_DEF(SQL_REDO, 12) +MINER_SCHEMA_DEF(SQL_UNDO, 13) +MINER_SCHEMA_DEF(ORG_CLUSTER_ID, 14) + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_analyzer.cpp b/src/logservice/logminer/ob_log_miner_analyzer.cpp new file mode 100644 index 0000000000..a7f2033317 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analyzer.cpp @@ -0,0 +1,239 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_analyzer.h" +#include "ob_log_miner_utils.h" +#include "ob_log_miner_br_producer.h" +#include "ob_log_miner_br_filter.h" +#include "ob_log_miner_br_converter.h" +#include "ob_log_miner_args.h" +#include "ob_log_miner_analysis_writer.h" +#include "ob_log_miner_data_manager.h" +#include "ob_log_miner_resource_collector.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +ObLogMinerAnalyzer::ObLogMinerAnalyzer(): + lib::ThreadPool(1), + is_inited_(false), + is_stopped_(false), + start_time_(-1), + producer_(nullptr), + data_filter_(nullptr), + data_converter_(nullptr), + writer_(nullptr), + data_manager_(nullptr), + resource_collector_(nullptr), + file_manager_(nullptr) { } + +ObLogMinerAnalyzer::~ObLogMinerAnalyzer() +{ + destroy(); +} + +int ObLogMinerAnalyzer::init(const AnalyzerArgs &args, ILogMinerFileManager *file_mgr) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("analyzer has been initialized", K(is_inited_)); + } else { + LOGMINER_STDOUT("ObLogMinerAnalyzer init...\n"); + file_manager_ = file_mgr; + if (OB_FAIL(lib::ThreadPool::init())) { + LOG_ERROR("analyzer thread pool failed to init"); + } else if (OB_FAIL(init_component(data_manager_, + LogMinerMode::ANALYSIS, args.record_format_, args.start_time_us_, args.end_time_us_, this))) { + LOG_ERROR("failed to init ObLogMinerDataManager", K(args)); + } else if (OB_FAIL(init_component(resource_collector_, + data_manager_, this))) { + LOG_ERROR("failed to init ObLogMinerResourceCollector", K(args)); + } else if (OB_FAIL(init_component(writer_, + args.start_time_us_, data_manager_, resource_collector_, file_manager_, this))) { + LOG_ERROR("failed to init ObLogMinerAnalysisWriter", K(args)); + } else if (OB_FAIL(init_component(data_converter_, + data_manager_, writer_, resource_collector_, this))) { + LOG_ERROR("failed to init ObLogMinerBRConverter", K(args)); + } else if (OB_FAIL(init_component(data_filter_, + args.column_cond_, args.operations_, data_manager_, resource_collector_, data_converter_, this))) { + LOG_ERROR("failed to init ObLogMinerBRFilter", K(args)); + } else if (OB_FAIL(init_component(producer_, args, data_filter_, this))) { + LOG_ERROR("failed to init ObLogMinerBRProducer", K(args)); + } else { + is_inited_ = true; + LOG_INFO("ObLogMinerAnalyzer finished to init", K(args), KP(file_mgr)); + LOGMINER_STDOUT("ObLogMinerAnalyzer finished to init\n"); + } + } + return ret; +} + +int ObLogMinerAnalyzer::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("analyzer hasn't been initialized", K(is_inited_)); + } else if (OB_FAIL(data_manager_->start())) { + LOG_ERROR("data_manager failed to start"); + } else if (OB_FAIL(resource_collector_->start())) { + LOG_ERROR("resource_collector failed to start"); + } else if (OB_FAIL(writer_->start())) { + LOG_ERROR("analyzer_writer failed to start"); + } else if (OB_FAIL(data_converter_->start())) { + LOG_ERROR("data_converter failed to start"); + } else if (OB_FAIL(data_filter_->start())) { + LOG_ERROR("data_filter failed to start"); + } else if (OB_FAIL(producer_->start())) { + LOG_ERROR("data_producer failed to start"); + } else if (OB_FAIL(lib::ThreadPool::start())) { + LOG_ERROR("analyzer thread failed to start"); + } else { + is_stopped_ = false; + start_time_ = ObClockGenerator::getClock(); + LOG_INFO("ObLogMinerAnalyzer starts"); + LOGMINER_STDOUT("ObLogMinerAnalyzer starts\n"); + } + + return ret; +} + +void ObLogMinerAnalyzer::stop() +{ + int ret = OB_SUCCESS; + double interval_sec = (double)(ObClockGenerator::getClock() - start_time_) / (double)1000000; + LOGMINER_STDOUT("ObLogMinerAnalyzer took %.2lfs to process\n", interval_sec); + LOGMINER_STDOUT("ObLogMinerAnalyzer exit...\n"); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("analyzer hasn't been initialized"); + } else { + lib::Threads::stop(); + data_filter_->stop(); + data_converter_->stop(); + writer_->stop(); + resource_collector_->stop(); + data_manager_->stop(); + // resource collector depends on producer + producer_->stop(); + ATOMIC_STORE(&is_stopped_, true); + stop_cond_.signal(); + LOG_INFO("ObLogMinerAnalyzer stopped"); + LOGMINER_STDOUT_V("ObLogMinerAnalyzer stopped\n"); + } +} + +void ObLogMinerAnalyzer::wait() +{ + if (IS_INIT) { + const int64_t wait_time_interval = 10L * 1000 * 1000; + while (! ATOMIC_LOAD(&is_stopped_)) { + stop_cond_.timedwait(wait_time_interval); + } + lib::Threads::wait(); + writer_->wait(); + data_converter_->wait(); + data_filter_->wait(); + data_manager_->wait(); + resource_collector_->wait(); + producer_->wait(); + } + LOG_INFO("ObLogMinerAnalyzer finished to wait"); + LOGMINER_STDOUT_V("ObLogMinerAnalyzer finished to wait\n"); +} + +void ObLogMinerAnalyzer::destroy() +{ + if (IS_INIT) { + is_inited_ = false; + start_time_ = -1; + lib::Threads::destroy(); + destroy_component(producer_); + destroy_component(data_filter_); + destroy_component(data_converter_); + destroy_component(writer_); + destroy_component(resource_collector_); + destroy_component(data_manager_); + file_manager_ = nullptr; + LOG_INFO("ObLogMinerAnalyzer destroyed"); + LOGMINER_STDOUT("ObLogMinerAnalyzer destroyed\n"); + } +} + +void ObLogMinerAnalyzer::handle_error(int err_code, const char *fmt, ...) +{ + int ret = OB_SUCCESS; + constexpr int64_t err_msg_len = 1024; + char err_msg[err_msg_len] = {0}; + static int stop_once = 0; + + if (IS_INIT) { + va_list arg; + ret = err_code; + va_start(arg, fmt); + vsnprintf(err_msg, err_msg_len, fmt, arg); + if (ret != OB_SUCCESS) { + LOG_ERROR(err_msg); + LOGMINER_STDOUT(err_msg); + LOGMINER_STDOUT("please check log[%s] for more detail\n", ObLogMinerArgs::LOGMINER_LOG_FILE); + } else { + LOG_INFO(err_msg); + } + va_end(arg); + if (ATOMIC_BCAS(&stop_once, 0, 1)) { + stop(); + } + } +} + +void ObLogMinerAnalyzer::run1() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("analyzer hasn't been initialized"); + } else { + const int64_t STAT_INTERVAL = 10L * 1000 * 1000; // 10s + const int64_t LOOP_INTERVAL = 1L * 1000 * 1000; // 1s + while (!has_set_stop()) { + if (REACH_TIME_INTERVAL(STAT_INTERVAL)) { + int64_t filter_task_cnt = 0; + int64_t converter_task_cnt = 0; + int64_t writer_task_cnt = 0; + int64_t rc_task_cnt = 0; + if (OB_FAIL(data_filter_->get_total_task_count(filter_task_cnt))) { + LOG_ERROR("failed to get task count in data_filter"); + } else if (OB_FAIL(data_converter_->get_total_task_count(converter_task_cnt))) { + LOG_ERROR("failed to get task count in data_converter"); + } else if (OB_FAIL(writer_->get_total_task_count(writer_task_cnt))) { + LOG_ERROR("failed to get task count in writer"); + } else if (OB_FAIL(resource_collector_->get_total_task_count(rc_task_cnt))) { + LOG_ERROR("failed to get task count in resource collector"); + } else { + LOG_INFO("[ANALYZER][TASK_STAT]", "Filter", filter_task_cnt, "Converter", converter_task_cnt, + "Writer", writer_task_cnt, "ResourceCollector", rc_task_cnt); + } + } + ob_usleep(LOOP_INTERVAL); + } + } +} + +} +} diff --git a/src/logservice/logminer/ob_log_miner_analyzer.h b/src/logservice/logminer/ob_log_miner_analyzer.h new file mode 100644 index 0000000000..ff5a002d71 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analyzer.h @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_ANALYZER_H_ +#define OCEANBASE_LOG_MINER_ANALYZER_H_ + +#include "ob_log_miner_error_handler.h" +#include "ob_queue_thread.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerBRProducer; +class ILogMinerBRFilter; +class ILogMinerBRConverter; +class ILogMinerAnalysisWriter; +class ILogMinerResourceCollector; +class ILogMinerDataManager; +class ILogMinerFileManager; +class AnalyzerArgs; + +class ObLogMinerAnalyzer: public ILogMinerErrorHandler, lib::ThreadPool { +public: + ObLogMinerAnalyzer(); + virtual ~ObLogMinerAnalyzer(); + int init(const AnalyzerArgs &args, ILogMinerFileManager *file_mgr); + int start(); + void stop(); + void wait(); + void destroy(); +public: + void handle_error(int err_code, const char *fmt, ...); + +public: + virtual void run1(); + +private: + bool is_inited_; + bool is_stopped_; + int64_t start_time_; + common::ObCond stop_cond_; + ILogMinerBRProducer *producer_; + ILogMinerBRFilter *data_filter_; + ILogMinerBRConverter *data_converter_; + ILogMinerAnalysisWriter *writer_; + ILogMinerDataManager *data_manager_; + ILogMinerResourceCollector *resource_collector_; + ILogMinerFileManager *file_manager_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_analyzer_checkpoint.cpp b/src/logservice/logminer/ob_log_miner_analyzer_checkpoint.cpp new file mode 100644 index 0000000000..3f59997784 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analyzer_checkpoint.cpp @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_analyzer_checkpoint.h" +#include "ob_log_miner_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +////////////////////////////// ObLogMinerCheckpoint ////////////////////////////// +const char * ObLogMinerCheckpoint::progress_key_str = "PROGRESS"; +const char * ObLogMinerCheckpoint::cur_file_id_key_str = "CUR_FILE_ID"; +const char * ObLogMinerCheckpoint::max_file_id_key_str = "MAX_FILE_ID"; + +int ObLogMinerCheckpoint::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n%s=%ld\n%s=%ld\n", + progress_key_str, progress_, cur_file_id_key_str, cur_file_id_, max_file_id_key_str, max_file_id_))) { + LOG_ERROR("failed to fill formatted checkpoint into buffer", K(buf_len), K(pos), KPC(this)); + } + return ret; +} + +int ObLogMinerCheckpoint::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(parse_line(progress_key_str, buf, data_len, pos, progress_))) { + LOG_ERROR("parse progress failed", K(progress_key_str), K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(cur_file_id_key_str, buf, data_len, pos, cur_file_id_))) { + LOG_ERROR("parse cur_file_id failed", K(cur_file_id_key_str), K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(max_file_id_key_str, buf, data_len, pos, max_file_id_))) { + LOG_ERROR("parse max_file_id failed", K(max_file_id_key_str), K(data_len), K(pos)); + } + + return ret; +} + +int64_t ObLogMinerCheckpoint::get_serialize_size() const +{ + int64_t len = 0; + const int64_t digit_max_len = 30; + char digit[digit_max_len] = {0}; + len += snprintf(digit, digit_max_len, "%ld", progress_); + len += snprintf(digit, digit_max_len, "%ld", cur_file_id_); + len += snprintf(digit, digit_max_len, "%ld", max_file_id_); + len += strlen(progress_key_str) + 2 + strlen(cur_file_id_key_str) + 2 + strlen(max_file_id_key_str) + 2; + return len; +} + + + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_analyzer_checkpoint.h b/src/logservice/logminer/ob_log_miner_analyzer_checkpoint.h new file mode 100644 index 0000000000..b9e28d8f8c --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_analyzer_checkpoint.h @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_ANALYZER_CHECKPOINT_H_ +#define OCEANBASE_LOG_MINER_ANALYZER_CHECKPOINT_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +struct ObLogMinerCheckpoint +{ +public: + ObLogMinerCheckpoint() { reset(); } + ~ObLogMinerCheckpoint() { reset(); } + + void reset() { + progress_ = OB_INVALID_TIMESTAMP; + max_file_id_ = -1; + cur_file_id_ = -1; + } + + ObLogMinerCheckpoint &operator=(const ObLogMinerCheckpoint &that) + { + progress_ = that.progress_; + max_file_id_ = that.max_file_id_; + cur_file_id_ = that.cur_file_id_; + return *this; + } + + bool operator==(const ObLogMinerCheckpoint &that) const + { + return progress_ == that.progress_ && cur_file_id_ == that.cur_file_id_ && max_file_id_ == that.max_file_id_; + } + + NEED_SERIALIZE_AND_DESERIALIZE; + + TO_STRING_KV( + K(progress_), + K(max_file_id_), + K(cur_file_id_) + ); +private: + static const char *progress_key_str; + static const char *cur_file_id_key_str; + static const char *max_file_id_key_str; + +public: + int64_t progress_; + int64_t max_file_id_; + int64_t cur_file_id_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_args.cpp b/src/logservice/logminer/ob_log_miner_args.cpp new file mode 100644 index 0000000000..e1328b348b --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_args.cpp @@ -0,0 +1,920 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "lib/container/ob_array_wrap.h" +#include "ob_log_miner_utils.h" // logminer_str2ll +#include "ob_log_miner_args.h" // ObLogMinerCmdArgs/ObLogMinerArgs/AnalyzerArgs/FlashbackerArgs +#include "ob_log_miner_logger.h" // LOGMINER_LOGGER +#include "lib/ob_define.h" // OB_SUCCESS +#include "lib/oblog/ob_log.h" // LOG_XX +#include "lib/timezone/ob_time_convert.h" // ObTimeConverter +#include "lib/timezone/ob_timezone_info.h" // ObTimeZoneInfo +#include // getopt_long + +namespace oceanbase +{ +namespace oblogminer +{ + +/////////////////////////////////////// ObLogMinerCmdArgs /////////////////////////////////////// + +void ObLogMinerCmdArgs::print_usage(const char *prog_name) +{ + fprintf(stderr, + "OceanBase LogMiner(ObLogMiner) is a tool that converts CLOG in OceanBase to\n" + "a human-readable format.\n" + "The following are examples demonstrating how to use:\n" + "%s -c \"ip:port\" -u \"user@tenant\" -p \"***\" -s \"2023-01-01 10:00:00\" -o \"file:///data/output/\"\n" + "%s -a \"file:///data/archive/\" -s \"2023-01-01 10:00:00\" -o \"file:///data/output/\"\n" + "Options are list below:\n" + " -m, --mode Specifies the working mode of ObLogMiner, which currently\n" + " only supports \"analysis\" mode. The default is \"analysis\".\n" + " -c, --cluster-addr A list of IPs and ports separated by '|' (e.g., ip1:port1|ip2:port2),\n" + " where the IP is the observer's IP and the port is the observer's SQL port.\n" + " -u, --user-name The tenant's username, including the tenant name(e.g., user@tenant).\n" + " -p, --password The password corresponding to the user-name.\n" + " -a, --archive-dest The location for the tenant's archive log(e.g., file:///data/xxx/xx).\n" + " -l, --table-list Specifies the table list for analysis, corresponding to\n" + " the `tb_white_list` parameter in OBCDC.\n" + " -C, --column-cond Specifies the column filter for analysis.\n" + " -s, --start-time Specifies the start time in microsecond timestamp or datetime format.\n" + " -e, --end-time Specifies the end time in microsecond timestamp or datetime format.\n" + " -O, --operations Specifies the type of operations, options including insert,\n" + " update, and delete. Operations can be combined using '|'.\n" + " The default is \"insert|update|delete\".\n" + " -o, --output The location for the analysis results.\n" + " -h, --help Displays the help message.\n" + " -v, --verbose Determines whether to output detailed logs to the command line.\n" + " -z, --timezone Specifies the timezone, default is \"+8:00\".\n" + " -f, --record_format Specifies the record format, default is \"CSV\".\n" + " Options including CSV, REDO_ONLY, UNDO_ONLY, JSON.\n" + " -L --log_level Specifies the log level, default is \"ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN\".\n" + "\n", + prog_name, prog_name + ); +} + +void ObLogMinerCmdArgs::reset() +{ + mode_ = LogMinerMode::UNKNOWN; + cluster_addr_ = nullptr; + user_name_ = nullptr; + password_ = nullptr; + archive_dest_ = nullptr; + table_list_ = nullptr; + column_cond_ = nullptr; + operations_ = nullptr; + output_ = nullptr; + recovery_path_ = nullptr; + target_table_ = nullptr; + source_table_ = nullptr; + start_time_ = nullptr; + end_time_ = nullptr; + log_level_ = nullptr; + timezone_ = nullptr; + record_format_ = nullptr; + verbose_ = false; + print_usage_ = false; +} + +int ObLogMinerCmdArgs::init(int argc, char **argv) +{ + int ret = OB_SUCCESS; + int opt = -1; + const char *short_opts = "m:c:u:p:a:l:C:s:e:O:o:r:t:S:hL:z:f:v"; + const struct option long_opts[] = { + {"mode", 1, NULL, 'm'}, + {"cluster-addr", 1, NULL, 'c'}, + {"user-name", 1, NULL, 'u'}, + {"password", 1, NULL, 'p'}, + {"archive-dest", 1, NULL, 'a'}, + {"table-list", 1, NULL, 'l'}, + {"column-cond", 1, NULL, 'C'}, + {"start-time", 1, NULL, 's'}, + {"end-time", 1, NULL, 'e'}, + {"operations", 1, NULL, 'O'}, + {"output", 1, NULL, 'o'}, + {"recovery-path", 1, NULL, 'r'}, + {"target-table", 1, NULL, 't'}, + {"source-table", 1, NULL, 'S'}, + {"help", 0, NULL, 'h'}, + {"log_level", 1, NULL, 'L'}, + {"timezone", 1, NULL, 'z'}, + {"record_format", 1, NULL, 'f'}, + {"verbose", 0, NULL, 'v'}, + {0, 0, 0, 0}, + }; + + if (argc <= 1) { + // print usage, just as -h + print_usage(argv[0]); + ret = OB_IN_STOP_STATE; + } + // set logminer default mode. + if (OB_SUCC(ret)) { + mode_ = LogMinerMode::ANALYSIS; + } + + while (OB_SUCC(ret) && (-1 != (opt = getopt_long(argc, argv, short_opts, long_opts, NULL)))) { + _LOG_TRACE("get opt %d, val is \"%s\"", opt, optarg); + switch (opt) { + case 'm': { + mode_ = get_logminer_mode(optarg); + break; + } + + case 'c': { + cluster_addr_ = optarg; + break; + } + + case 'u': { + user_name_ = optarg; + break; + } + + case 'p': { + password_ = optarg; + break; + } + + case 'a': { + archive_dest_ = optarg; + break; + } + + case 'l': { + table_list_ = optarg; + break; + } + + case 'C': { + column_cond_ = optarg; + break; + } + + case 's': { + start_time_ = optarg; + break; + } + + case 'e': { + end_time_ = optarg; + break; + } + + case 'o': { + output_ = optarg; + break; + } + + case 'O': { + operations_ = optarg; + break; + } + + case 'r': { + recovery_path_ = optarg; + break; + } + + case 't': { + target_table_ = optarg; + break; + } + + case 'S': { + source_table_ = optarg; + break; + } + + case 'h': { + print_usage_ = true; + break; + } + + case 'L': { + log_level_ = optarg; + break; + } + + case 'z': { + timezone_ = optarg; + break; + } + + case 'f': { + record_format_ = optarg; + break; + } + + case 'v': { + verbose_ = true; + break; + } + + default: { + ret = OB_INVALID_ARGUMENT; + break; + } + } + } + LOG_INFO("parse cmd args finished", KPC(this)); + if (OB_SUCC(ret) && OB_FAIL(validate_())) { + LOG_ERROR("logminer command arguments are invalid", KPC(this)); + LOGMINER_STDOUT("logminer command arguments are invalid\n"); + } + + return ret; +} + +int ObLogMinerCmdArgs::validate_() const +{ + int ret = OB_SUCCESS; + if (print_usage_) { + // do nothing + } else if (! is_logminer_mode_valid(mode_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("logminer mode is invalid", K(mode_)); + LOGMINER_STDOUT("logminer mode is invalid\n"); + } else if (is_analysis_mode(mode_)) { + const bool server_args_set = (nullptr != cluster_addr_) || (nullptr != user_name_) || (nullptr != password_); + const bool server_args_complete = (nullptr != cluster_addr_) && (nullptr != user_name_) && (nullptr != password_); + const bool archive_args_set = (nullptr != archive_dest_); + const bool is_other_required_args_set = (nullptr != start_time_) && (nullptr != output_); + const bool is_flashback_spec_args_set = (nullptr != recovery_path_) || (nullptr != target_table_) || + (nullptr != source_table_); + if (archive_args_set && server_args_set) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("server_args and archive_args are both set", K(server_args_set), K(archive_args_set)); + LOGMINER_STDOUT("(cluster_addr, user_name, password) and (archive_dest) are both specified\n"); + } else if (!server_args_complete && !archive_args_set) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("server_args and archive_args are both empty or incomplete", K(server_args_set), K(archive_args_set)); + LOGMINER_STDOUT("(cluster_addr, user_name, password) and (archive_dest) are both empty or incomplete\n"); + } else if (! is_other_required_args_set) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("start_time or output must be set in analysis mode"); + LOGMINER_STDOUT("start_time or output are empty\n"); + } else if (is_flashback_spec_args_set) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("some flashback specified arguments are set in analysis mode", + KCSTRING(target_table_), KCSTRING(source_table_)); + LOGMINER_STDOUT("some flashback specified arguments are set in analysis mode\n"); + } + } else if (is_flashback_mode(mode_)) { + const bool server_args_set = (nullptr != cluster_addr_) && (nullptr != user_name_) && (nullptr != password_); + const bool is_other_required_args_set = (nullptr != recovery_path_) && + (nullptr != target_table_) && (nullptr != source_table_) && + (nullptr != start_time_) && (nullptr != end_time_) && + (nullptr != output_); + const bool is_analysis_spec_args_set = (nullptr != table_list_) || (nullptr != operations_) || + (nullptr != archive_dest_); + + if (!server_args_set || !is_other_required_args_set) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("some of the required arguments are not set in flashback mode", KCSTRING(cluster_addr_), KCSTRING(user_name_), + KCSTRING(target_table_), KCSTRING(source_table_), KCSTRING(start_time_), KCSTRING(end_time_)); + } else if (is_analysis_spec_args_set) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("some analysis specified arguments are set in flashback mode", KCSTRING(table_list_), KCSTRING(operations_)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("logminer mode is invalid", K(mode_)); + LOGMINER_STDOUT("logminer mode is invalid\n"); + } + + return ret; +} + +/////////////////////////////////////// AnalyzerArgs /////////////////////////////////////// + +const char *AnalyzerArgs::CLUSTER_ADDR_KEY = "cluster_addr"; +const char *AnalyzerArgs::USER_NAME_KEY = "user_name"; +const char *AnalyzerArgs::PASSWORD_KEY = "password"; +const char *AnalyzerArgs::ARCHIVE_DEST_KEY = "archive_dest"; +const char *AnalyzerArgs::TABLE_LIST_KEY = "table_list"; +const char *AnalyzerArgs::COLUMN_COND_KEY = "column_cond"; +const char *AnalyzerArgs::OPERATIONS_KEY = "operations"; +const char *AnalyzerArgs::OUTPUT_DST_KEY = "output_dest"; +const char *AnalyzerArgs::LOG_LEVEL_KEY = "log_level"; +const char *AnalyzerArgs::START_TIME_US_KEY = "start_time_us"; +const char *AnalyzerArgs::END_TIME_US_KEY = "end_time_us"; +const char *AnalyzerArgs::TIMEZONE_KEY = "timezone"; +const char *AnalyzerArgs::RECORD_FORMAT_KEY = "record_format"; +void AnalyzerArgs::reset() +{ + cluster_addr_ = nullptr; + user_name_ = nullptr; + password_ = nullptr; + archive_dest_ = nullptr; + table_list_ = nullptr; + column_cond_ = nullptr; + operations_ = nullptr; + output_dst_ = nullptr; + log_level_ = nullptr; + start_time_us_ = OB_INVALID_TIMESTAMP; + end_time_us_ = OB_INVALID_TIMESTAMP; + timezone_ = nullptr; + record_format_ = RecordFileFormat::INVALID; + alloc_.reset(); +} + +int AnalyzerArgs::init(const ObLogMinerCmdArgs &args) +{ + int ret = OB_SUCCESS; + if (! is_analysis_mode(args.mode_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("incorrect cmd args to init analyzer args", K(args)); + } else { + const char *operations = nullptr == args.operations_ ? ObLogMinerArgs::DEFAULT_LOGMNR_OPERATIONS : args.operations_; + const char *timezone = nullptr == args.timezone_ ? ObLogMinerArgs::DEFAULT_LOGMNR_TIMEZONE : args.timezone_; + const char *table_list = nullptr == args.table_list_ ? ObLogMinerArgs::DEFAULT_LOGMNR_TABLE_LIST : args.table_list_; + const char *record_format_str = nullptr == args.record_format_ ? ObLogMinerArgs::DEFAULT_LOGMNR_FORMAT : args.record_format_; + ObTimeZoneInfo tz_info; + ObTimeConvertCtx cvrt_ctx(&tz_info, true); + if (OB_FAIL(deep_copy_cstring(alloc_, args.cluster_addr_, cluster_addr_))) { + LOG_ERROR("failed to deep copy cluster addr", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, args.user_name_, user_name_))) { + LOG_ERROR("failed to deep copy user name", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, args.password_, password_))) { + LOG_ERROR("failed to deep copy password", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, args.archive_dest_, archive_dest_))) { + LOG_ERROR("failed to deep copy archive dest", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, table_list, table_list_))) { + LOG_ERROR("failed to deep copy table list", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, args.column_cond_, column_cond_))) { + LOG_ERROR("failed to deep copy column cond", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, operations, operations_))) { + LOG_ERROR("failed to deep copy operations", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, args.output_, output_dst_))) { + LOG_ERROR("failed to deep copy output dest", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, args.log_level_, log_level_))) { + LOG_ERROR("failed to deep copy log level", K(args)); + } else if (OB_FAIL(deep_copy_cstring(alloc_, timezone, timezone_))) { + LOG_ERROR("failed to deep copy timezone", K(args)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(tz_info.set_timezone(ObString(timezone_)))) { + LOG_ERROR("parse timezone failed", K(ret), KCSTRING(timezone_)); + LOGMINER_STDOUT("parse timezone failed\n"); + } else { + record_format_ = get_record_file_format(record_format_str); + if (RecordFileFormat::INVALID == record_format_) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("parse record_format failed", K(ret), KCSTRING(record_format_str)); + LOGMINER_STDOUT("parse record_format failed\n"); + } + } + } + if (OB_SUCC(ret)) { + if (is_number(args.start_time_)) { + if (OB_FAIL(logminer_str2ll(args.start_time_, start_time_us_))) { + LOG_ERROR("parse start time for AnalyzerArgs failed", K(ret), K(args), + K(start_time_us_), K(errno), K(strerror(errno))); + } + } else if (OB_FAIL(ObTimeConverter::str_to_datetime(ObString(args.start_time_), cvrt_ctx, start_time_us_))) { + LOG_ERROR("parse start time for AnalyzerArgs failed", K(ret), K(args), K(start_time_us_)); + } + if (OB_FAIL(ret)) { + LOGMINER_STDOUT("parse start time for AnalyzerArgs failed\n"); + } + } + if (OB_SUCC(ret)) { + if (nullptr == args.end_time_) { + end_time_us_ = ObTimeUtility::current_time(); + } else if (is_number(args.end_time_)) { + if (OB_FAIL(logminer_str2ll(args.end_time_, end_time_us_))) { + LOG_ERROR("parse end time for AnalyzerArgs failed", K(ret), K(args), K(end_time_us_), K(errno), + K(strerror(errno))); + } + } else if (OB_FAIL(ObTimeConverter::str_to_datetime(ObString(args.end_time_), cvrt_ctx, end_time_us_))) { + LOG_ERROR("parse end time for AnalyzerArgs failed", K(ret), K(args), K(end_time_us_)); + } + if (OB_FAIL(ret)) { + LOGMINER_STDOUT("parse end time for AnalyzerArgs failed\n"); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(validate())) { + LOG_ERROR("validate AnalyzerArgs failed after initialize all arguments", K(args), KPC(this)); + } else { + // succ + LOG_INFO("finished to init analyzer_args", KPC(this)); + } + } + } + return ret; +} + +int AnalyzerArgs::validate() const +{ + int ret = OB_SUCCESS; + bool is_server_avail = (cluster_addr_ != nullptr) && (user_name_ != nullptr) && (password_ != nullptr); + bool is_server_arg_set = (cluster_addr_ != nullptr) || (user_name_ != nullptr) || (password_ != nullptr); + bool is_archive_avail = (archive_dest_ != nullptr); + if ((is_archive_avail && is_server_arg_set) || (!is_archive_avail && !is_server_avail)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("(cluster_addr, user_name, password) and (archive_dest) are both" + " specified or not completely specified", K(is_server_avail), K(is_archive_avail)); + LOGMINER_STDOUT("(cluster_addr, user_name, password) and (archive_dest) are both" + " specified or not completely specified\n"); + } else if (OB_INVALID_TIMESTAMP == start_time_us_ || + OB_INVALID_TIMESTAMP == end_time_us_ || + start_time_us_ >= end_time_us_) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("start_time or end_time is not valid or start_time exceeds end_time", + K(start_time_us_), K(end_time_us_)); + LOGMINER_STDOUT("start_time or end_time is not valid or start_time exceeds end_time\n"); + } else if (nullptr == output_dst_) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("output_dst is null which is not expected", KPC(this)); + LOGMINER_STDOUT("output_dst should be specified\n"); + } else { + const char *user_name_delim = "@#"; + char *user_name_buf = nullptr; + char *strtok_ptr = nullptr; + char *user = nullptr; + char *tenant = nullptr; + int user_name_size = 0; + if (is_server_avail) { + // only support user tenant + user_name_size = strlen(user_name_) + 1; + if (OB_ISNULL(user_name_buf = reinterpret_cast(ob_malloc(user_name_size, "LogMnrUsrBuf")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("err alloc string buffer", K(ret), K(user_name_size)); + } else { + MEMCPY(user_name_buf, user_name_, user_name_size-1); + user_name_buf[user_name_size-1] = '\0'; + } + user = STRTOK_R(user_name_buf, user_name_delim, &strtok_ptr); + tenant = STRTOK_R(nullptr, user_name_delim, &strtok_ptr); + if (nullptr == tenant || 0 == strcmp(tenant, OB_SYS_TENANT_NAME)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("oblogminer only support user tenant(e.g., user@tenant).", K(tenant)); + LOGMINER_STDOUT("oblogminer only support user tenant(e.g., user@tenant).\n"); + } + // check password not empty + if (OB_SUCC(ret)) { + if (0 == strcmp(password_, "")) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("password should not be empty string", KCSTRING(password_)); + LOGMINER_STDOUT("password should not be empty string\n"); + } + } + } + if (OB_SUCC(ret) && nullptr != tenant) { + bool validate = true; + if (OB_FAIL(validate_table_list_(tenant, validate))) { + LOG_ERROR("validate table list failed", K(ret), KCSTRING(tenant), KCSTRING(table_list_)); + LOGMINER_STDOUT("parse table_list failed\n"); + } else { + if (!validate) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("tenant in table_list must match the tenant specified in user-name", KCSTRING(tenant)); + LOGMINER_STDOUT("tenant in table_list must match the tenant specified in user-name\n"); + } + } + } + if (OB_NOT_NULL(user_name_buf)) { + ob_free(user_name_buf); + } + } + + return ret; +} + +// check tenant name in table_list +int AnalyzerArgs::validate_table_list_(const char *tenant_name, bool &validate) const +{ + int ret = OB_SUCCESS; + const char pattern_delimiter = '|'; + const char name_delimiter = '.'; + bool done = false; + int buffer_size = strlen(table_list_) + 1; + char *buffer = nullptr; + ObString remain; + ObString cur_pattern; + ObString tenant_pattern; + ObString database_pattern; + ObString table_pattern; + if (OB_ISNULL(buffer = reinterpret_cast(ob_malloc(buffer_size, "LogMnrTableList")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("err alloc string buffer", K(ret), K(buffer_size)); + } else { + MEMCPY(buffer, table_list_, buffer_size-1); + buffer[buffer_size-1] = '\0'; + remain.assign_ptr(buffer, buffer_size); + } + while (OB_SUCCESS == ret && !done) { + // Split Pattern & get current pattern. + cur_pattern = remain.split_on(pattern_delimiter); + if (cur_pattern.empty()) { + cur_pattern = remain; + done = true; + } + if (OB_SUCC(ret)) { + tenant_pattern = cur_pattern.split_on(name_delimiter); + if (tenant_pattern.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid argment", K(ret), K(cur_pattern)); + break; + } + if (OB_SUCC(ret)) { + if (0 != tenant_pattern.compare(tenant_name) && 0 != tenant_pattern.compare("*")) { + validate = false; + LOG_INFO("tenant in table_list must match the tenant specified in user-name", + KCSTRING(tenant_name), K(tenant_pattern)); + break; + } + } + } + if (OB_SUCC(ret)) { + database_pattern = cur_pattern.split_on(name_delimiter); + if (database_pattern.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid argument", KR(ret), K(cur_pattern)); + break; + } + } + // Table name. + if (OB_SUCC(ret)) { + table_pattern = cur_pattern; + if (table_pattern.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid argment", KR(ret), K(cur_pattern)); + break; + } + } + } // while + if (OB_NOT_NULL(buffer)) { + ob_free(buffer); + } + return ret; +} + +int AnalyzerArgs::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + // make sure there is no nullptr among class members + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", CLUSTER_ADDR_KEY, + empty_str_wrapper(cluster_addr_)))) { + LOG_ERROR("failed to print cluster_addr", KCSTRING(cluster_addr_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", USER_NAME_KEY, + empty_str_wrapper(user_name_)))) { + LOG_ERROR("failed to print user_name", KCSTRING(user_name_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", PASSWORD_KEY, + empty_str_wrapper(password_)))) { + LOG_ERROR("failed to print password", K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", ARCHIVE_DEST_KEY, + empty_str_wrapper(archive_dest_)))) { + LOG_ERROR("failed to print archive_dest", K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", TABLE_LIST_KEY, + empty_str_wrapper(table_list_)))) { + LOG_ERROR("failed to print table_list", KCSTRING(table_list_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", COLUMN_COND_KEY, + empty_str_wrapper(column_cond_)))) { + LOG_ERROR("failed to print column cond", KCSTRING(column_cond_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", OPERATIONS_KEY, + empty_str_wrapper(operations_)))) { + LOG_ERROR("failed to print operations", KCSTRING(operations_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", OUTPUT_DST_KEY, + empty_str_wrapper(output_dst_)))) { + LOG_ERROR("failed to print output_dst", K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", LOG_LEVEL_KEY, + empty_str_wrapper(log_level_)))) { + LOG_ERROR("failed to print log_level", KCSTRING(log_level_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n", START_TIME_US_KEY, + start_time_us_))) { + LOG_ERROR("failed to print start_time_us", K(start_time_us_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n", END_TIME_US_KEY, + end_time_us_))) { + LOG_ERROR("failed to print end_time_us", K(end_time_us_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", TIMEZONE_KEY, + timezone_))) { + LOG_ERROR("failed to print timezone", KCSTRING(timezone_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%s\n", RECORD_FORMAT_KEY, + record_file_format_str(record_format_)))) { + LOG_ERROR("failed to print record format", KCSTRING(record_file_format_str(record_format_)), + K(buf_len), K(pos)); + } + return ret; +} + +int AnalyzerArgs::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + ObString record_format_str; + if (OB_FAIL(parse_line(CLUSTER_ADDR_KEY, buf, data_len, pos, alloc_, cluster_addr_))) { + LOG_ERROR("failed to get cluster_addr", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(USER_NAME_KEY, buf, data_len, pos, alloc_, user_name_))) { + LOG_ERROR("failed to get user_name", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(PASSWORD_KEY, buf, data_len, pos, alloc_, password_))) { + LOG_ERROR("failed to get password", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(ARCHIVE_DEST_KEY, buf, data_len, pos, alloc_, archive_dest_))) { + LOG_ERROR("failed to get archive_dest", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(TABLE_LIST_KEY, buf, data_len, pos, alloc_, table_list_))) { + LOG_ERROR("failed to get table_list", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(COLUMN_COND_KEY, buf, data_len, pos, alloc_, column_cond_))) { + LOG_ERROR("failed to get column_cond", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(OPERATIONS_KEY, buf, data_len, pos, alloc_, operations_))) { + LOG_ERROR("failed to get operations", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(OUTPUT_DST_KEY, buf, data_len, pos, alloc_, output_dst_))) { + LOG_ERROR("failed to get output_dst", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(LOG_LEVEL_KEY, buf, data_len, pos, alloc_, log_level_))) { + LOG_ERROR("failed to get log_level", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(START_TIME_US_KEY, buf, data_len, pos, start_time_us_))) { + LOG_ERROR("failed to get start_time_us", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(END_TIME_US_KEY, buf, data_len, pos, end_time_us_))) { + LOG_ERROR("failed to get end_time_us", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(TIMEZONE_KEY, buf, data_len, pos, alloc_, timezone_))) { + LOG_ERROR("failed to get timezone", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(RECORD_FORMAT_KEY, buf, data_len, pos, record_format_str))) { + LOG_ERROR("failed to ger record_format", K(data_len), K(pos)); + } + if (OB_SUCC(ret)) { + record_format_ = get_record_file_format(record_format_str); + } + return ret; +} + +int64_t AnalyzerArgs::get_serialize_size() const +{ + int64_t size = 0; + char buf[30] = {0}; + int64_t start_time_us_len = snprintf(buf, sizeof(buf), "%ld", start_time_us_); + int64_t end_time_us_len = snprintf(buf, sizeof(buf), "%ld", end_time_us_); + // need add "=" and "\n" + size += strlen(CLUSTER_ADDR_KEY) + 1 + strlen(empty_str_wrapper(cluster_addr_)) + 1; + size += strlen(USER_NAME_KEY) + 1 + strlen(empty_str_wrapper(user_name_)) + 1; + size += strlen(PASSWORD_KEY) + 1 + strlen(empty_str_wrapper(password_)) + 1; + size += strlen(ARCHIVE_DEST_KEY) + 1 + strlen(empty_str_wrapper(archive_dest_)) + 1; + size += strlen(TABLE_LIST_KEY) + 1 + strlen(empty_str_wrapper(table_list_)) + 1; + size += strlen(COLUMN_COND_KEY) + 1 + strlen(empty_str_wrapper(column_cond_)) + 1; + size += strlen(OPERATIONS_KEY) + 1 + strlen(empty_str_wrapper(operations_)) + 1; + size += strlen(OUTPUT_DST_KEY) + 1 + strlen(empty_str_wrapper(output_dst_)) + 1; + size += strlen(LOG_LEVEL_KEY) + 1 + strlen(empty_str_wrapper(log_level_)) + 1; + size += strlen(START_TIME_US_KEY) + 1 + start_time_us_len + 1; + size += strlen(END_TIME_US_KEY) + 1 + end_time_us_len + 1; + size += strlen(TIMEZONE_KEY) + 1 + strlen(empty_str_wrapper(timezone_)) + 1; + size += strlen(RECORD_FORMAT_KEY) + 1 + strlen(record_file_format_str(record_format_)) + 1; + return size; +} + +/////////////////////////////////////// FlashbackerArgs /////////////////////////////////////// + +void FlashbackerArgs::reset() +{ + cluster_addr_ = nullptr; + user_name_ = nullptr; + password_ = nullptr; + recovery_path_ = nullptr; + tgt_table_ = nullptr; + src_table_ = nullptr; + progress_dst_ = nullptr; + log_level_ = nullptr; + start_time_us_ = OB_INVALID_TIMESTAMP; + end_time_us_ = OB_INVALID_TIMESTAMP; +} + +int FlashbackerArgs::init(const ObLogMinerCmdArgs &args) +{ + int ret = OB_SUCCESS; + if (!is_flashback_mode(args.mode_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("incorrect cmd args to init flashbacker args", K(args)); + } else { + cluster_addr_ = args.cluster_addr_; + user_name_ = args.user_name_; + password_ = args.password_; + recovery_path_ = args.recovery_path_; + tgt_table_ = args.target_table_; + src_table_ = args.source_table_; + progress_dst_ = args.output_; + log_level_ = args.log_level_; + + if (OB_FAIL(logminer_str2ll(args.start_time_, start_time_us_))) { + LOG_ERROR("parse start time for FlashbackerArgs failed", K(args), K(start_time_us_)); + } else if (OB_FAIL(logminer_str2ll(args.end_time_, end_time_us_))) { + LOG_ERROR("parse end time for FlashbackerArgs failed", K(args), K(end_time_us_)); + } else if (OB_FAIL(validate())) { + LOG_ERROR("validate FlashbackerArgs failed after initialize all arguments", K(args), KPC(this)); + } else { + // succ + LOG_INFO("finished to init flashbacker args", KPC(this)); + } + } + + return ret; +} + +int FlashbackerArgs::validate() const +{ + int ret = OB_SUCCESS; + + if (nullptr == cluster_addr_ || nullptr == user_name_ || nullptr == password_) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("cluster_addr, user_name or password is empty", KPC(this)); + } else if (nullptr == tgt_table_ || nullptr == src_table_) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("target-table or source-table is empty", KPC(this)); + } else if (nullptr == progress_dst_ || nullptr == recovery_path_) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("recovery path or progress_dst is empty", KPC(this)); + } else if (OB_INVALID_TIMESTAMP == start_time_us_ || + OB_INVALID_TIMESTAMP == end_time_us_ || + start_time_us_ >= end_time_us_) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("start_time or end_time is not valid", K(start_time_us_), K(end_time_us_), KPC(this)); + } else { + // succ + } + + return ret; +} + +int FlashbackerArgs::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + // TODO + return ret; +} + +int FlashbackerArgs::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + // TODO + return ret; +} + +int64_t FlashbackerArgs::get_serialize_size() const +{ + int64_t size = 0; + // TODO + return size; +} + +/////////////////////////////////////// ObLogMinerArgs /////////////////////////////////////// +const char *ObLogMinerArgs::DEFAULT_LOG_LEVEL = "ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN"; +const char *ObLogMinerArgs::LOGMINER_LOG_FILE = "oblogminer.log"; +const char *ObLogMinerArgs::DEFAULT_LOGMNR_TIMEZONE = "+8:00"; +const char *ObLogMinerArgs::DEFAULT_LOGMNR_OPERATIONS = "insert|delete|update"; +const char *ObLogMinerArgs::DEFAULT_LOGMNR_TABLE_LIST = "*.*.*"; +const char *ObLogMinerArgs::DEFAULT_LOGMNR_FORMAT = "csv"; + + +void ObLogMinerArgs::reset() +{ + mode_ = LogMinerMode::UNKNOWN; + verbose_ = false; + print_usage_ = false; + timezone_ = nullptr; + analyzer_args_.reset(); + flashbacker_args_.reset(); +} + +int ObLogMinerArgs::validate() const +{ + int ret = OB_SUCCESS; + if (! is_logminer_mode_valid(mode_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("logminer mode is invalid", K(mode_)); + } else if (is_analysis_mode(mode_)) { + ret = analyzer_args_.validate(); + } else if (is_flashback_mode(mode_)) { + ret = flashbacker_args_.validate(); + } + + if (OB_FAIL(ret)) { + LOG_ERROR("ObLogMiner found some arguments invalid", KPC(this)); + } + + return ret; +} + +int ObLogMinerArgs::init(int argc, char *argv[]) +{ + int ret = OB_SUCCESS; + ObLogMinerCmdArgs args; + if (OB_FAIL(args.init(argc, argv))) { + LOG_ERROR("init ObLogMinerCmdArgs failed", K(argc)); + } else if (args.print_usage_) { + print_usage_ = args.print_usage_; + } else if (! is_logminer_mode_valid(args.mode_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("ObLogMinerArgs got invalid mode argument", K(args.mode_), + "mode", logminer_mode_to_str(args.mode_)); + } else { + mode_ = args.mode_; + verbose_ = args.verbose_; + timezone_ = args.timezone_ == nullptr ? DEFAULT_LOGMNR_TIMEZONE : args.timezone_; + + if (nullptr == args.log_level_) { + args.log_level_ = DEFAULT_LOG_LEVEL; + } + + if (OB_FAIL(ret)) { + LOG_ERROR("failed to set mod log level previously, unexpected"); + } else if (is_analysis_mode(args.mode_)) { + if (OB_FAIL(analyzer_args_.init(args))) { + LOG_ERROR("failed to ini analyzer args", K(args.mode_), K(analyzer_args_), K(args)); + } + } else if (is_flashback_mode(args.mode_)) { + if (OB_FAIL(flashbacker_args_.init(args))) { + LOG_ERROR("failed to ini flashbacker args", K(args.mode_), K(flashbacker_args_), K(args)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("vaild logminer mode is ignored, which is unexpected", K(args.mode_), + "mode", logminer_mode_to_str(args.mode_)); + } + } + + return ret; +} + +int ObLogMinerArgs::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf + pos, buf_len, pos, "mode=%s\n", logminer_mode_to_str(mode_)))) { + LOG_ERROR("failed to serialize mode", K(mode_)); + } else if (is_analysis_mode(mode_)) { + if (OB_FAIL(analyzer_args_.serialize(buf, buf_len, pos))) { + LOG_ERROR("analyzer_args failed to serialize", K(buf), K(buf_len), K(pos), K(analyzer_args_)); + } + } else if (is_flashback_mode(mode_)) { + if (OB_FAIL(flashbacker_args_.serialize(buf, buf_len, pos))) { + LOG_ERROR("flashbacker_args failed to serialize", K(buf), K(buf_len), K(pos), K(flashbacker_args_)); + } + } + return ret; +} + +int ObLogMinerArgs::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + ObString mode_str; + if (OB_FAIL(parse_line("mode", buf, data_len, pos, mode_str))) { + LOG_ERROR("failed to get mode string", K(data_len), K(pos)); + } else { + mode_ = get_logminer_mode(mode_str); + if (is_analysis_mode(mode_)) { + if (OB_FAIL(analyzer_args_.deserialize(buf, data_len, pos))) { + LOG_ERROR("failed to deserialize analyzer arguments", K(data_len), K(pos)); + } + } else { + ret = OB_INVALID_DATA; + LOG_ERROR("get invalid mode when deserializing logminer args", K(mode_str), K(mode_)); + } + } + return ret; +} + +int64_t ObLogMinerArgs::get_serialize_size() const +{ + int64_t size = 0; + + size += strlen("mode=") + strlen(logminer_mode_to_str(mode_)) + strlen("\n"); + + if (is_analysis_mode(mode_)) { + size += analyzer_args_.get_serialize_size(); + } else if (is_flashback_mode(mode_)) { + size += flashbacker_args_.get_serialize_size(); + } + + return size; +} + + +int64_t ObLogMinerArgs::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + databuff_print_kv(buf, buf_len, pos, "mode", mode_); + if (is_analysis_mode(mode_)) { + databuff_print_kv(buf, buf_len, pos, "analyzer_args", analyzer_args_); + } else if (is_flashback_mode(mode_)) { + databuff_print_kv(buf, buf_len, pos, "flashbacker_args", flashbacker_args_); + } else { + // invalid mode + } + return pos; +} + + + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_args.h b/src/logservice/logminer/ob_log_miner_args.h new file mode 100644 index 0000000000..f36187363d --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_args.h @@ -0,0 +1,201 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCENABASE_LOG_MINER_ARGS_H_ +#define OCENABASE_LOG_MINER_ARGS_H_ + +#include +#include "lib/allocator/page_arena.h" // ObArenaAllocator +#include "lib/string/ob_string_buffer.h" +#include "lib/utility/ob_print_utils.h" +#include "ob_log_miner_mode.h" +#include "ob_log_miner_record_file_format.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerCmdArgs { +public: + static void print_usage(const char *prog_name); +public: + ObLogMinerCmdArgs() + { reset(); } + + int init(int argc, char *argv[]); + + void reset(); + + TO_STRING_KV( + K(mode_), + KCSTRING(cluster_addr_), + KCSTRING(user_name_), + KCSTRING(table_list_), + KCSTRING(column_cond_), + KCSTRING(operations_), + KCSTRING(target_table_), + KCSTRING(source_table_), + KCSTRING(start_time_), + KCSTRING(end_time_), + KCSTRING(log_level_), + KCSTRING(timezone_), + KCSTRING(record_format_), + K(verbose_), + K(print_usage_) + ); + +public: + LogMinerMode mode_; + const char *cluster_addr_; + const char *user_name_; + const char *password_; + const char *archive_dest_; + const char *table_list_; + const char *column_cond_; + const char *operations_; + const char *output_; + const char *recovery_path_; + const char *target_table_; + const char *source_table_; + const char *start_time_; + const char *end_time_; + const char *log_level_; + const char *timezone_; + const char *record_format_; + bool verbose_; + bool print_usage_; +private: + int validate_() const; +}; + +class AnalyzerArgs { + static const char *CLUSTER_ADDR_KEY; + static const char *USER_NAME_KEY; + static const char *PASSWORD_KEY; + static const char *ARCHIVE_DEST_KEY; + static const char *TABLE_LIST_KEY; + static const char *COLUMN_COND_KEY; + static const char *OPERATIONS_KEY; + static const char *OUTPUT_DST_KEY; + static const char *LOG_LEVEL_KEY; + static const char *START_TIME_US_KEY; + static const char *END_TIME_US_KEY; + static const char *TIMEZONE_KEY; + static const char *RECORD_FORMAT_KEY; +public: + AnalyzerArgs() { reset(); } + int init(const ObLogMinerCmdArgs &args); + int validate() const; + void reset(); + + NEED_SERIALIZE_AND_DESERIALIZE; + + TO_STRING_KV( + KCSTRING(cluster_addr_), + KCSTRING(user_name_), + KCSTRING(table_list_), + KCSTRING(column_cond_), + KCSTRING(operations_), + KCSTRING(log_level_), + K(start_time_us_), + K(end_time_us_), + KCSTRING(timezone_), + K(record_format_) + ); + +public: + char *cluster_addr_; + char *user_name_; + char *password_; + char *archive_dest_; + char *table_list_; + char *column_cond_; + char *operations_; + char *output_dst_; + char *log_level_; + int64_t start_time_us_; + int64_t end_time_us_; + char *timezone_; + RecordFileFormat record_format_; +private: + int validate_table_list_(const char *tenant_name, bool &validate) const; +private: + ObArenaAllocator alloc_; +}; + +class FlashbackerArgs { +public: + FlashbackerArgs() { reset(); } + int init(const ObLogMinerCmdArgs &args); + int validate() const; + void reset(); + + NEED_SERIALIZE_AND_DESERIALIZE; + + TO_STRING_KV( + KCSTRING(cluster_addr_), + KCSTRING(user_name_), + KCSTRING(tgt_table_), + KCSTRING(src_table_), + KCSTRING(progress_dst_), + KCSTRING(log_level_), + K(start_time_us_), + K(end_time_us_) + ); +public: + const char *cluster_addr_; + const char *user_name_; + const char *password_; + const char *recovery_path_; + const char *tgt_table_; + const char *src_table_; + const char *progress_dst_; + const char *log_level_; + int64_t start_time_us_; + int64_t end_time_us_; +}; + +class ObLogMinerArgs { +public: + static const char *DEFAULT_LOG_LEVEL; + static const char *LOGMINER_LOG_FILE; + static const char *DEFAULT_LOGMNR_TIMEZONE; + static const char *DEFAULT_LOGMNR_OPERATIONS; + static const char *DEFAULT_LOGMNR_TABLE_LIST; + static const char *DEFAULT_LOGMNR_FORMAT; +public: + ObLogMinerArgs() { reset(); } + ~ObLogMinerArgs() { reset(); } + int init(int argc, char *argv[]); + int validate() const; + void reset(); + + NEED_SERIALIZE_AND_DESERIALIZE; + + DECLARE_TO_STRING; + +public: + LogMinerMode mode_; + bool verbose_; + bool print_usage_; + const char *timezone_; + AnalyzerArgs analyzer_args_; + FlashbackerArgs flashbacker_args_; +private: + DISALLOW_COPY_AND_ASSIGN(ObLogMinerArgs); +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_batch_record.cpp b/src/logservice/logminer/ob_log_miner_batch_record.cpp new file mode 100644 index 0000000000..a49f398c06 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_batch_record.cpp @@ -0,0 +1,219 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_record_converter.h" +#include "ob_log_miner_batch_record.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +void ObLogMinerBatchRecord::reset() +{ + range_.reset(); + begin_record_.reset(); + buf_.reset(); + trans_data_record_count_ = 0; + total_data_record_count_ = 0; + written_record_count_ = 0; + first_access_ts_ = OB_INVALID_TIMESTAMP; + last_trans_end_ts_ = OB_INVALID_TIMESTAMP; + + seq_no_ = -1; + file_id_ = -1; + file_offset_ = -1; + is_file_end_ = false; + + freezed_ = false; +} + +int ObLogMinerBatchRecord::init_last_trans_end_ts(const int64_t commit_ts) +{ + int ret = OB_SUCCESS; + + if (OB_INVALID_TIMESTAMP == last_trans_end_ts_) { + last_trans_end_ts_ = commit_ts; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("last trans end ts rollbacked, unexpected", K(last_trans_end_ts_), K(commit_ts)); + } + + return ret; +} + +int ObLogMinerBatchRecord::append_record(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + if (freezed_) { + ret = OB_BUF_NOT_ENOUGH; + LOG_WARN("batch record has been freezed, need retry", K(freezed_), "buf_len", buf_.length(), + "buf_cap", buf_.capacity()); + } else { + const RecordType record_type = record.get_record_type(); + if (OB_INVALID_TIMESTAMP == first_access_ts_) { + first_access_ts_ = ObTimeUtility::current_time(); + } + switch (record_type) { + case EBEGIN: { + ret = handle_begin_record_(record); + break; + } + + case ECOMMIT: { + ret = handle_commit_record_(record); + break; + } + + case EDDL: { + ret = handle_ddl_record_(record); + break; + } + + case EINSERT: + case EUPDATE: + case EDELETE: { + ret = handle_dml_record_(record); + break; + } + + case HEARTBEAT: { + ret = handle_heartbeat_record_(record); + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get unexpected record type", K(record)); + } + } + + if (OB_FAIL(ret)) { + LOG_ERROR("append record to buffer failed", K(record)); + } else { + const int64_t record_commit_ts = record.get_commit_scn().convert_to_ts(); + update_progress_range_(record_commit_ts); + total_data_record_count_++; + if (is_trans_end_record_type(record_type)) { + last_trans_end_ts_ = record_commit_ts; + } + if (buf_.length() >= BATCH_RECORD_THRESHOLD) { + freezed_ = true; + } + } + } + + + return ret; +} + +int ObLogMinerBatchRecord::handle_begin_record_(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + if (begin_record_.is_inited()) { + ret = OB_ERR_DEFENSIVE_CHECK; + LOG_ERROR("get duplicate begin record, unexpected", K(begin_record_), K(record), KPC(this)); + } else { + begin_record_.copy_base_info(record); + } + return ret; +} + +int ObLogMinerBatchRecord::handle_commit_record_(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + if (begin_record_.is_inited() && 0 == trans_data_record_count_) { + // exit, no need to write begin & commit record + LOG_TRACE("all data in trans is filtered", K(record), K(begin_record_), KPC(this)); + begin_record_.reset(); + } else if (OB_FAIL(write_record_(record))) { + LOG_ERROR("failed to write commit record into buffer", K(record), KPC(this)); + } else { + trans_data_record_count_ = 0; + + if (begin_record_.is_inited()) { + begin_record_.reset(); + } + } + + return ret; +} + +int ObLogMinerBatchRecord::handle_dml_record_(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + if (begin_record_.is_inited()) { + if (OB_FAIL(write_record_(begin_record_))) { + LOG_ERROR("write begin record into buffer failed", K(begin_record_), KPC(this)); + } else { + begin_record_.reset(); + } + } + + if (OB_SUCC(ret) && OB_FAIL(write_record_(record))) { + LOG_ERROR("failed to write dml record", K(record), KPC(this)); + } else { + trans_data_record_count_++; + } + return ret; +} + +int ObLogMinerBatchRecord::handle_ddl_record_(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(write_record_(record))) { + LOG_ERROR("write ddl record failed", K(record), KPC(this)); + } + return ret; +} + +int ObLogMinerBatchRecord::handle_heartbeat_record_(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + + return ret; +} + +void ObLogMinerBatchRecord::update_progress_range_(const int64_t progress) +{ + if (OB_INVALID_TIMESTAMP == range_.min_commit_ts_) { + range_.min_commit_ts_ = progress; + } + + if (OB_INVALID_TIMESTAMP == range_.max_commit_ts_ || progress >= range_.max_commit_ts_){ + range_.max_commit_ts_ = progress; + } +} + +int ObLogMinerBatchRecord::write_record_(const ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + bool is_written = false; + if (OB_ISNULL(converter_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get an invalid converter when write_record", K(record)); + } else if (OB_FAIL(converter_->write_record(record, buf_, is_written))) { + LOG_ERROR("converter failed to convert record", K(record), "buf_len", buf_.length(), + "buf_cap", buf_.capacity()); + } else { + if (is_written) { + written_record_count_++; + } + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_batch_record.h b/src/logservice/logminer/ob_log_miner_batch_record.h new file mode 100644 index 0000000000..ec0520d76e --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_batch_record.h @@ -0,0 +1,200 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_BATCH_RECORD_ +#define OCEANBASE_LOG_MINER_BATCH_RECORD_ + +#include "lib/allocator/ob_allocator.h" +#include "lib/string/ob_string_buffer.h" +#include "ob_log_miner_record.h" +#include "ob_log_miner_file_manager.h" +#include + +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerRecordConverter; + +class ObLogMinerBatchRecord: public ObLogMinerRecyclableTask +{ +public: + static const int64_t BATCH_RECORD_THRESHOLD = 2L * 1024 * 1024; // 2MB +private: + ObLogMinerBatchRecord(): + ObLogMinerRecyclableTask(TaskType::BATCH_RECORD), + alloc_(nullptr), + converter_(nullptr), + range_(), + begin_record_(), + buf_(), + trans_data_record_count_(0), + total_data_record_count_(0), + written_record_count_(0), + first_access_ts_(OB_INVALID_TIMESTAMP), + last_trans_end_ts_(OB_INVALID_TIMESTAMP), + seq_no_(-1), + file_id_(-1), + file_offset_(-1), + is_file_end_(false), + freezed_(false) + {} +public: + ObLogMinerBatchRecord(ObIAllocator *alloc, ILogMinerRecordConverter *converter): + ObLogMinerBatchRecord() + { + set_allocator(alloc); + set_record_converter(converter); + } + ~ObLogMinerBatchRecord() + { + reset(); + converter_ = nullptr; + alloc_ = nullptr; + } + + void reset(); + + void set_allocator(ObIAllocator *alloc) + { + alloc_ = alloc; + buf_.set_allocator(alloc_); + } + + int init_last_trans_end_ts(const int64_t commit_ts); + + int64_t get_last_trans_end_ts() const + { + return last_trans_end_ts_; + } + + void set_record_converter(ILogMinerRecordConverter *converter) + { + converter_ = converter; + } + + int append_record(const ObLogMinerRecord &record); + void get_data(const char *&data, int64_t &data_len) const { + data = buf_.ptr(); + data_len = buf_.length(); + } + + int64_t get_total_record_count() const { + return total_data_record_count_; + } + + int64_t get_cur_trans_record_count() const { + return trans_data_record_count_; + } + + int64_t get_written_record_count() const { + return written_record_count_; + } + + const ObLogMinerProgressRange &get_progress_range() const { + return range_; + } + int64_t get_first_access_ts() const { + return first_access_ts_; + } + bool is_freezed() const { + return freezed_; + } + + void set_seq_no(const int64_t seq_no) { + seq_no_ = seq_no; + } + + int64_t get_seq_no() const { + return seq_no_; + } + + void set_file_id(const int64_t file_id) { + file_id_ = file_id; + } + + int64_t get_file_id() const { + return file_id_; + } + + void set_file_offset(const int64_t file_offset) { + file_offset_ = file_offset; + } + + int64_t get_file_offset() const { + return file_offset_; + } + + void set_file_end() { + is_file_end_ = true; + } + + bool is_file_end() const { + return is_file_end_; + } + + TO_STRING_KV( + K(range_), + K(begin_record_), + K(trans_data_record_count_), + K(total_data_record_count_), + K(written_record_count_), + K(first_access_ts_), + K(seq_no_), + K(file_id_), + K(file_offset_), + K(is_file_end_), + K(freezed_) + ); + +private: + int handle_begin_record_(const ObLogMinerRecord &record); + + int handle_dml_record_(const ObLogMinerRecord &record); + + int handle_ddl_record_(const ObLogMinerRecord &record); + + int handle_commit_record_(const ObLogMinerRecord &record); + + int handle_heartbeat_record_(const ObLogMinerRecord &record); + + int write_record_(const ObLogMinerRecord &record); + + void update_progress_range_(const int64_t progress); + +private: + common::ObIAllocator *alloc_; + ILogMinerRecordConverter *converter_; + + ObLogMinerProgressRange range_; + ObLogMinerRecord begin_record_; + ObStringBuffer buf_; + int64_t trans_data_record_count_; + int64_t total_data_record_count_; + int64_t written_record_count_; + int64_t first_access_ts_; // in milliseconds + + int64_t last_trans_end_ts_; // in milliseconds + + int64_t seq_no_; + int64_t file_id_; + int64_t file_offset_; + bool is_file_end_; + + bool freezed_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_batch_record_writer.cpp b/src/logservice/logminer/ob_log_miner_batch_record_writer.cpp new file mode 100644 index 0000000000..4b1c08ed18 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_batch_record_writer.cpp @@ -0,0 +1,383 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "lib/container/ob_ext_ring_buffer.h" +#include "ob_log_miner_batch_record_writer.h" +#include "ob_log_miner_batch_record.h" +#include "ob_log_miner_file_manager.h" +#include "ob_log_miner_resource_collector.h" +#include "ob_log_miner_data_manager.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +ObLogMinerBatchRecordWriter::ObLogMinerBatchRecordWriter(): + is_inited_(false), + current_file_id_(-1), + current_file_offset_(-1), + batch_record_sw_(), + record_count_(0), + batch_record_count_(0), + data_manager_(nullptr), + resource_collector_(nullptr), + file_manager_(nullptr), + err_handle_(nullptr) { } + +int ObLogMinerBatchRecordWriter::init(ILogMinerDataManager *data_mgr, + ILogMinerResourceCollector *res_collector, + ILogMinerFileManager *file_mgr, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + const int64_t SLIDING_WINDOW_SIZE = 2L << 20; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("BatchRecordWriter has been initialized", K(is_inited_)); + } else if (OB_ISNULL(data_mgr) || OB_ISNULL(res_collector) || OB_ISNULL(file_mgr)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid argument when initializing BatchRecordWriter", K(data_mgr), + K(res_collector), K(file_mgr)); + } else if (OB_FAIL(BatchWriterThreadPool::init(BATCH_WRITER_THREAD_NUM, BATCH_WRITER_QUEUE_SIZE))) { + LOG_ERROR("BatchWriterThreadPool failed to init"); + } else if (OB_FAIL(batch_record_sw_.init(1, SLIDING_WINDOW_SIZE))) { + LOG_ERROR("batch record sliding window failed to initialize", K(SLIDING_WINDOW_SIZE)); + } else { + current_file_id_ = 0; + current_file_offset_ = 0; + data_manager_ = data_mgr; + resource_collector_ = res_collector; + file_manager_ = file_mgr; + err_handle_ = err_handle; + is_inited_ = true; + LOG_INFO("ObLogMinerBatchRecordWriter finished to init", K(data_manager_), K(resource_collector_), + K(file_manager_)); + } + + return ret; +} + +int ObLogMinerBatchRecordWriter::start() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(BatchWriterThreadPool::start())) { + LOG_ERROR("BatchWriterThreadPool failed to start"); + } else if (OB_FAIL(lib::ThreadPool::start())) { + LOG_ERROR("thread pool in batchrecord writer failed to start"); + } else { + LOG_INFO("ObLogMinerBatchRecordWriter starts"); + } + + return ret; +} + +void ObLogMinerBatchRecordWriter::stop() +{ + BatchWriterThreadPool::mark_stop_flag(); + lib::ThreadPool::stop(); + LOG_INFO("ObLogMinerBatchRecordWriter stopped"); +} + +void ObLogMinerBatchRecordWriter::wait() +{ + BatchWriterThreadPool::stop(); + lib::ThreadPool::wait(); + LOG_INFO("ObLogMinerBatchRecordWriter finished to wait"); +} + +void ObLogMinerBatchRecordWriter::destroy() +{ + if (IS_INIT) { + BatchWriterThreadPool::destroy(); + lib::ThreadPool::destroy(); + current_file_id_ = -1; + current_file_offset_ = -1; + data_manager_ = nullptr; + resource_collector_ = nullptr; + file_manager_ = nullptr; + err_handle_ = nullptr; + record_count_ = 0; + batch_record_count_ = 0; + batch_record_sw_.destroy(); + is_inited_ = false; + } +} + + +// only one thread will call this method +int ObLogMinerBatchRecordWriter::push(ObLogMinerBatchRecord *record) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_)); + } else if (OB_ISNULL(record)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid record when trying to push record into BatchWriter", K(record)); + } else { + LOG_TRACE("ObLogMinerBatchRecordWriter push batch record into queue", KPC(record)); + const int64_t DATA_OP_TIMEOUT = 1L * 1000 * 1000; + static volatile int64_t seq_no = 0; + const int64_t assigned_seq_no = ATOMIC_AAF(&seq_no, 1); + if (OB_FAIL(set_file_info_(*record, assigned_seq_no))) { + LOG_ERROR("set file info for record failed", K(record), K(current_file_id_), K(current_file_offset_)); + } else { + ATOMIC_AAF(&record_count_, record->get_total_record_count()); + ATOMIC_INC(&batch_record_count_); + RETRY_FUNC(is_stoped(), *(static_cast(this)), push, record, record->get_file_id(), DATA_OP_TIMEOUT); + } + } + + return ret; +} + +int ObLogMinerBatchRecordWriter::handle(void *data, const int64_t thread_index, volatile bool &stop_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_)); + } else if (OB_ISNULL(data)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid data when handling data in BatchWriter", K(data)); + } else { + ObLogMinerBatchRecord *record = static_cast(data); + const char *data = nullptr; + int64_t data_len = 0; + record->get_data(data, data_len); + if (OB_FAIL(file_manager_->append_records(*record))) { + LOG_ERROR("write file through file manager failed", KPC(record), K(data_len)); + } else if (OB_FAIL(generate_meta_flush_task_(*record))) { + LOG_ERROR("generate meta flush task failed", K(record)); + } else { + ATOMIC_SAF(&record_count_, record->get_total_record_count()); + ATOMIC_DEC(&batch_record_count_); + if(OB_FAIL(resource_collector_->revert(record))) { + LOG_ERROR("resource collecter failed to revert record", K(record)); + } + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "ObLogMinerBatchRecordWriter exit unexpected\n"); + } + + return ret; +} + +int ObLogMinerBatchRecordWriter::get_total_record_count(int64_t &record_count) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_)); + } else { + record_count = ATOMIC_LOAD(&record_count_); + } + + return ret; +} + +int ObLogMinerBatchRecordWriter::get_total_batch_record_count(int64_t &batch_record_count) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_)); + } else { + batch_record_count = ATOMIC_LOAD(&batch_record_count_); + } + + return ret; +} + +void ObLogMinerBatchRecordWriter::run1() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_)); + } else { + const int64_t META_FLUSH_INTERVAL = 1000L * 1000; // 1s + while (OB_SUCC(ret) && ! lib::Threads::has_set_stop()) { + if (OB_FAIL(proceed_checkpoint_())) { + LOG_ERROR("logminer batchrecord writer failed to proceed checkpoint"); + } + ob_usleep(META_FLUSH_INTERVAL); + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "Checkpointer exit unexpected\n"); + } +} + +int ObLogMinerBatchRecordWriter::set_file_info_(ObLogMinerBatchRecord &record, + const int64_t seq_no) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("BatchRecordWriter hasn't been initialized", K(is_inited_)); + } else { + const char *unused_data = nullptr; + const int64_t record_max_commit_ts = record.get_progress_range().max_commit_ts_; + const int64_t cur_trans_record_count = record.get_cur_trans_record_count(); + int64_t data_len = 0; + record.get_data(unused_data, data_len); + + record.set_seq_no(seq_no); + record.set_file_id(current_file_id_); + record.set_file_offset(current_file_offset_); + + if (0 >= cur_trans_record_count && + data_manager_->reach_end_progress(record_max_commit_ts)) { + record.set_file_end(); + LOG_TRACE("max commit ts of batch record exceed the end_progress and there is no " + "pending transaction", K(record_max_commit_ts), K(cur_trans_record_count)); + } + + // invariant: current_file_offset_ < FILE_SPLIT_THRESHOLD + ATOMIC_AAF(¤t_file_offset_, data_len); + + if (current_file_offset_ >= FILE_SPLIT_THRESHOLD) { + ATOMIC_INC(¤t_file_id_); + ATOMIC_STORE(¤t_file_offset_, 0); + record.set_file_end(); + } + + } + + return ret; +} + +int ObLogMinerBatchRecordWriter::generate_meta_flush_task_(const ObLogMinerBatchRecord &record) +{ + int ret = OB_SUCCESS; + BatchRecordMetaTask *task = reinterpret_cast(ob_malloc( + sizeof(BatchRecordMetaTask), "LogMnrMetaTask")); + if (OB_ISNULL(task)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("meta task is null, allocate memory failed", K(task)); + } else { + task = new(task) BatchRecordMetaTask(); + task->file_id_ = record.get_file_id(); + task->is_file_complete_ = record.is_file_end(); + task->last_trans_end_ts_ = record.get_last_trans_end_ts(); + task->record_cnt_ = record.get_written_record_count(); + LOG_TRACE("generated meta flush task", KPC(task), "seq_no", record.get_seq_no()); + if (OB_FAIL(batch_record_sw_.set(record.get_seq_no(), task))) { + LOG_ERROR("set task into batch_record sliding window failed", K(record)); + } + } + + return ret; +} + +int ObLogMinerBatchRecordWriter::proceed_checkpoint_() +{ + int ret = OB_SUCCESS; + BatchRecordMetaTaskPopFunc func; + BatchRecordMetaTask *last_poped_task = nullptr, *curr_task = nullptr; + ObLogMinerCheckpoint ckpt; + bool popped = true; + constexpr int64_t PRINT_INTERVAL = 10L * 1000 * 1000; + + while (OB_SUCC(ret) && popped && OB_SUCC(batch_record_sw_.pop(func, curr_task, popped))) { + // continuity check + if (popped && nullptr != curr_task) { + if (OB_INVALID_TIMESTAMP == ckpt.progress_ || ckpt.progress_ <= curr_task->last_trans_end_ts_) { + ckpt.progress_ = curr_task->last_trans_end_ts_; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("the progress of meta_flush task rollbacked", K(ckpt), KPC(curr_task)); + } + + if (-1 == ckpt.cur_file_id_ || ckpt.cur_file_id_ <= curr_task->file_id_) { + ckpt.cur_file_id_ = curr_task->file_id_; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("the file id of meta_flush task rollbacked", K(ckpt), KPC(curr_task)); + } + + if (curr_task->is_file_complete_) { + if (-1 == ckpt.max_file_id_ || ckpt.max_file_id_ == curr_task->file_id_ - 1) { + ckpt.max_file_id_ = curr_task->file_id_; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("max file id of meta_flush task not expected", K(ckpt), KPC(curr_task)); + } + } + + if (OB_SUCC(ret) && OB_FAIL(data_manager_->increase_record_count(curr_task->record_cnt_))) { + LOG_ERROR("failed to increase record count", K(curr_task->record_cnt_)); + } + + LOG_TRACE("task poped", KPC(curr_task)); + if (nullptr != last_poped_task) { + ob_free(last_poped_task); + } + // cannot access last_poped_task afterwards + last_poped_task = curr_task; + } else { + last_poped_task = nullptr; + } + } + + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("failed to proceed checkpoint", K(curr_task)); + } else { + ret = OB_SUCCESS; + } + + if (OB_SUCC(ret)) { + if (nullptr != curr_task) { + ckpt.cur_file_id_ = curr_task->file_id_; + ckpt.progress_ = curr_task->last_trans_end_ts_; + ckpt.max_file_id_ = curr_task->is_file_complete_ ? curr_task->file_id_ : curr_task->file_id_ - 1; + + if (OB_FAIL(file_manager_->write_checkpoint(ckpt))) { + LOG_ERROR("file_manager failed to write checkpoint", K(ckpt), K(curr_task)); + } else if (OB_FAIL(data_manager_->update_output_progress(ckpt.progress_))) { + LOG_ERROR("failed to update output progress", K(ckpt)); + } else { + if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) { + LOG_INFO("succeed to proceed checkpoint", K(ckpt), K(curr_task)); + } + } + + if (OB_SUCC(ret)) { + // cannot access curr_task afterwards + LOG_TRACE("proceed checkpoint", K(ckpt), KPC(curr_task)); + ob_free(curr_task); + } + } else { + // only one thread + if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) { + LOG_INFO("try to proceed checkpoint but no batch record meta task", K(popped)); + } + } + } + + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_batch_record_writer.h b/src/logservice/logminer/ob_log_miner_batch_record_writer.h new file mode 100644 index 0000000000..0bef7f99d0 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_batch_record_writer.h @@ -0,0 +1,131 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_BATCH_WRITER_H_ +#define OCEANBASE_LOG_MINER_RECORD_BATCH_WRITER_H_ + +#include "lib/thread/ob_multi_fixed_queue_thread.h" +#include "lib/container/ob_ext_ring_buffer.h" +#include "ob_log_miner_file_manager.h" +#include "ob_log_miner_error_handler.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerBatchRecord; +class ILogMinerDataManager; +class ILogMinerResourceCollector; +class ILogMinerFileManager; + +class ILogMinerBatchRecordWriter +{ +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int push(ObLogMinerBatchRecord *record) = 0; + virtual int get_total_record_count(int64_t &record_count) = 0; + virtual int get_total_batch_record_count(int64_t &batch_record_count) = 0; +}; + +struct BatchRecordMetaTask +{ + BatchRecordMetaTask() {reset();} + ~BatchRecordMetaTask() { reset(); } + void reset() { + file_id_ = -1; + last_trans_end_ts_ = OB_INVALID_TIMESTAMP; + record_cnt_ = 0; + is_file_complete_ = false; + } + + TO_STRING_KV( + K(file_id_), + K(last_trans_end_ts_), + K(record_cnt_), + K(is_file_complete_) + ) + + int64_t file_id_; + int64_t last_trans_end_ts_; + int64_t record_cnt_; + bool is_file_complete_; +}; + +struct BatchRecordMetaTaskPopFunc +{ + bool operator()(int64_t sn, BatchRecordMetaTask *task) { + UNUSED(sn); + return nullptr != task; + } +}; + +typedef common::ObMQThread<1, ILogMinerBatchRecordWriter> BatchWriterThreadPool; +typedef ObExtendibleRingBuffer BatchRecordSlidingWindow; +class ObLogMinerBatchRecordWriter: public ILogMinerBatchRecordWriter, BatchWriterThreadPool, lib::ThreadPool +{ +public: + static const int64_t BATCH_WRITER_THREAD_NUM = 1L; + static const int64_t BATCH_WRITER_QUEUE_SIZE = 100000L; + static const int64_t FILE_SPLIT_THRESHOLD = (64L << 20); // 64 MB +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int push(ObLogMinerBatchRecord *record); + virtual int get_total_record_count(int64_t &record_count); + virtual int get_total_batch_record_count(int64_t &batch_record_count); +public: + ObLogMinerBatchRecordWriter(); + ~ObLogMinerBatchRecordWriter() { + destroy(); + } + + virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag); + + virtual void run1(); + + int init(ILogMinerDataManager *data_mgr, + ILogMinerResourceCollector *res_collector, + ILogMinerFileManager *file_mgr, + ILogMinerErrorHandler *err_handle); + +private: + int set_file_info_(ObLogMinerBatchRecord &record, + const int64_t seq_no); + + int generate_meta_flush_task_(const ObLogMinerBatchRecord &record); + + int proceed_checkpoint_(); + +private: + bool is_inited_; + int64_t current_file_id_; + int64_t current_file_offset_; + BatchRecordSlidingWindow batch_record_sw_; + + int64_t record_count_; + int64_t batch_record_count_; + ILogMinerDataManager *data_manager_; + ILogMinerResourceCollector *resource_collector_; + ILogMinerFileManager *file_manager_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br.cpp b/src/logservice/logminer/ob_log_miner_br.cpp new file mode 100644 index 0000000000..2947583553 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br.cpp @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_binlog_record.h" +#include "ob_log_miner_br.h" +#include "lib/ob_define.h" +#include "lib/oblog/ob_log.h" + +namespace oceanbase +{ + +namespace oblogminer +{ + +int ObLogMinerBR::init(libobcdc::IObCDCInstance *host, + ICDCRecord *br, + const lib::Worker::CompatMode mode, + const transaction::ObTransID &trans_id, + const uint64_t tenant_id, + const int32_t major_version) +{ + int ret = OB_SUCCESS; + libobcdc::ObLogBR *oblog_br = nullptr; + share::SCN commit_scn; + if (nullptr == br) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get null binlog record", K(br), K(tenant_id), K(major_version)); + } else if (OB_INVALID_TENANT_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid tenant_id", K(br), K(tenant_id), K(major_version)); + } else if (OB_ISNULL(oblog_br = static_cast(br->getUserData()))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get ICDCRecord without user data, unexpected", KP(oblog_br)); + } else if (OB_FAIL(commit_scn.convert_for_inner_table_field(oblog_br->get_commit_version()))) { + LOG_ERROR("failed to convert commit_version to scn", "commit_version", oblog_br->get_commit_version()); + } else { + br_host_ = host; + br_ = br; + compat_mode_ = mode; + tenant_id_ = tenant_id; + major_version_ = major_version; + commit_scn_ = commit_scn; + record_type_ = static_cast(br->recordType()); + trans_id_ = trans_id; + } + return ret; +} + +void ObLogMinerBR::reset() +{ + is_filtered_ = false; + compat_mode_ = lib::Worker::CompatMode::INVALID; + seq_no_ = 0; + tenant_id_ = OB_INVALID_TENANT_ID; + major_version_ = 0; + commit_scn_.reset(); + record_type_ = RecordType::EUNKNOWN; + trans_id_ = transaction::ObTransID(); + if (nullptr != br_host_ && nullptr != br_) { + br_host_->release_record(br_); + } + br_host_ = nullptr; + br_ = nullptr; +} + +} + +} diff --git a/src/logservice/logminer/ob_log_miner_br.h b/src/logservice/logminer/ob_log_miner_br.h new file mode 100644 index 0000000000..09e27b6048 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br.h @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_BR_H_ +#define OCEANBASE_LOG_MINER_BR_H_ + +#include "libobcdc.h" +#include "lib/worker.h" +#include "ob_log_miner_recyclable_task.h" +#include "storage/tx/ob_trans_define.h" + +#ifndef OB_USE_DRCMSG +#include "ob_cdc_msg_convert.h" +#else +#include // ICDCRecord +#include // ITableMeta +#include // IStrArray +#include // binlogBuf +#include +#endif + +namespace oceanbase +{ +namespace oblogminer +{ +class ObLogMinerBR: public ObLogMinerRecyclableTask +{ +public: + ObLogMinerBR(): + ObLogMinerRecyclableTask(TaskType::BINLOG_RECORD), + is_filtered_(false), + compat_mode_(lib::Worker::CompatMode::INVALID), + seq_no_(0), + tenant_id_(OB_INVALID_TENANT_ID), + major_version_(0), + commit_scn_(), + record_type_(EUNKNOWN), + trans_id_(), + br_host_(nullptr), + br_(nullptr) + { + } + + ~ObLogMinerBR() { + destroy(); + } + + int init(libobcdc::IObCDCInstance *host, + ICDCRecord *br, + const lib::Worker::CompatMode mode, + const transaction::ObTransID &trans_id, + const uint64_t tenant_id, + const int32_t major_version); + + // reentrant + void reset(); + + void destroy() { + reset(); + } + + ICDCRecord *get_br() { + return br_; + } + + void mark_filtered() { + is_filtered_ = true; + } + + bool is_filtered() const { + return is_filtered_; + } + + lib::Worker::CompatMode get_compat_mode() const { + return compat_mode_; + } + + uint64_t get_tenant_id() const { + return tenant_id_; + } + + share::SCN get_commit_scn() const { + return commit_scn_; + } + + void set_seq_no(int64_t seq_no) { + seq_no_ = seq_no; + } + + int64_t get_seq_no() const { + return seq_no_; + } + + RecordType get_record_type() const { + return record_type_; + } + + const transaction::ObTransID& get_trans_id() const { + return trans_id_; + } + + TO_STRING_KV( + K(is_filtered_), + K(compat_mode_), + K(seq_no_), + K(tenant_id_), + K(commit_scn_), + K(record_type_), + K(trans_id_) + ); + +private: + bool is_filtered_; + lib::Worker::CompatMode compat_mode_; + int64_t seq_no_; + uint64_t tenant_id_; + int32_t major_version_; + share::SCN commit_scn_; + RecordType record_type_; + transaction::ObTransID trans_id_; + libobcdc::IObCDCInstance *br_host_; + ICDCRecord *br_; +}; +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br_converter.cpp b/src/logservice/logminer/ob_log_miner_br_converter.cpp new file mode 100644 index 0000000000..c6a2fbfa73 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br_converter.cpp @@ -0,0 +1,202 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_br_converter.h" +#include "ob_log_miner_resource_collector.h" +#include "ob_log_miner_data_manager.h" +#include "ob_log_miner_analysis_writer.h" +#include "ob_log_miner_record.h" +#include "ob_log_miner_br.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ +const int64_t ObLogMinerBRConverter::PUSH_BR_TIMEOUT_TIME = 1L * 1000 * 1000; +const int64_t ObLogMinerBRConverter::BR_CONVERTER_THREAD_NUM = 1L; +const int64_t ObLogMinerBRConverter::BR_CONVERTER_QUEUE_SIZE = 100000L; + +ObLogMinerBRConverter::ObLogMinerBRConverter(): + is_inited_(false), + data_manager_(nullptr), + writer_(nullptr), + resource_collector_(nullptr), + err_handle_(nullptr) { } + +ObLogMinerBRConverter::~ObLogMinerBRConverter() +{ + destroy(); +} + +int ObLogMinerBRConverter::init(ILogMinerDataManager *data_manager, + ILogMinerAnalysisWriter *writer, + ILogMinerResourceCollector *resource_collector, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("br converter has already been initialized", K(is_inited_)); + } else if (OB_ISNULL(data_manager) || OB_ISNULL(writer) || OB_ISNULL(resource_collector)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null components pointer", K(data_manager), K(writer), K(resource_collector)); + } else if (OB_FAIL(BRConverterThreadPool::init(BR_CONVERTER_THREAD_NUM, BR_CONVERTER_QUEUE_SIZE))) { + LOG_ERROR("BRConverterThreads failed to init"); + } else { + data_manager_ = data_manager; + writer_ = writer; + resource_collector_ = resource_collector; + err_handle_ = err_handle; + is_inited_ = true; + LOG_INFO("ObLogMinerBRConverter finished to init", K(data_manager_), K(writer_), + K(resource_collector_)); + LOGMINER_STDOUT_V("ObLogMinerBRConverter finished to init\n"); + } + return ret; +} + +int ObLogMinerBRConverter::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("logminer br converter is not initialized", K(is_inited_)); + } else if (OB_FAIL(BRConverterThreadPool::start())) { + LOG_ERROR("logminer br converter failed to start"); + } else { + // start + LOG_INFO("ObLogMinerBRConverter starts"); + } + return ret; +} + +void ObLogMinerBRConverter::stop() +{ + BRConverterThreadPool::mark_stop_flag(); + LOG_INFO("ObLogMinerBRConverter stopped"); +} + +void ObLogMinerBRConverter::wait() +{ + BRConverterThreadPool::stop(); + LOG_INFO("ObLogMinerBRConverter finished to wait"); +} + +void ObLogMinerBRConverter::destroy() +{ + if (IS_INIT) { + data_manager_ = nullptr; + writer_ = nullptr; + resource_collector_ = nullptr; + err_handle_ = nullptr; + BRConverterThreadPool::destroy(); + is_inited_ = false; + LOG_INFO("ObLogMinerBRConverter destroyed"); + LOGMINER_STDOUT_V("ObLogMinerBRConverter destroyed\n"); + } +} + +int ObLogMinerBRConverter::push(ObLogMinerBR *logminer_br) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRConverter is not inited", K(logminer_br)); + } else if (OB_ISNULL(logminer_br)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get a null logminer br in br converter", K(logminer_br)); + } else { + LOG_TRACE("BRConverter push logminer_br into queue", KPC(logminer_br)); + int64_t hash_val = 0; + RETRY_FUNC(is_stoped(), *(static_cast(this)), push, logminer_br, hash_val, PUSH_BR_TIMEOUT_TIME); + if (OB_SUCC(ret)) { + if (is_stoped()) { + ret = OB_IN_STOP_STATE; + } + } else { + LOG_ERROR("push logminer br into the queue of ObMQThread failed", K(logminer_br), K(hash_val)); + } + } + + + return ret; +} + +int ObLogMinerBRConverter::get_total_task_count(int64_t &record_count) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRConverter is not inited", K(this)); + } else if (OB_FAIL(BRConverterThreadPool::get_total_task_num(record_count))) { + LOG_ERROR("failed to get record count"); + } + + return ret; +} + +int ObLogMinerBRConverter::handle(void *data, const int64_t thread_index, volatile bool &stop_flag) +{ + int ret = OB_SUCCESS; + ObLogMinerBR *br = static_cast(data); + ObLogMinerRecord *record = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRConverter is not inited", K(br), K(is_inited_)); + } else if (OB_ISNULL(br)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get null br when handle br converter tasks", K(br)); + } else if (br->is_filtered()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("filtered br appears in converter", K(br)); + } else if (OB_FAIL(data_manager_->get_logminer_record(record))) { + LOG_ERROR("get logminer record from data manager failed", K(record), K(data_manager_)); + } else if (OB_ISNULL(record)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get a null record from data_manager", K(record), K(data_manager_)); + } else if (OB_FAIL(generate_logminer_record_(*br, *record))) { + LOG_ERROR("failed to generate logminer record", K(br), K(record)); + } else if (OB_FAIL(writer_->push(record))) { + LOG_ERROR("failed to push record into writer", K(record), K(writer_)); + } + if (OB_SUCC(ret) && OB_NOT_NULL(br)) { + if (OB_FAIL(resource_collector_->revert(br))) { + LOG_ERROR("failed to revert logminer br", K(br), K(resource_collector_)); + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "ObLogMinerBRFilter exit unexpected\n"); + } + return ret; +} + +int ObLogMinerBRConverter::generate_logminer_record_(ObLogMinerBR &br, + ObLogMinerRecord &record) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(record.init(br))) { + LOG_ERROR("record failed to init using logminerbr", K(br), K(record)); + } else if (record.is_ddl_record() || record.is_dml_record()) { + if (OB_FAIL(record.build_stmts(br))) { + LOG_ERROR("failed to build stmts for record", K(record), K(br)); + } + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br_converter.h b/src/logservice/logminer/ob_log_miner_br_converter.h new file mode 100644 index 0000000000..9d0eb2cb1f --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br_converter.h @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_BR_CONVERTER_H_ +#define OCEANBASE_LOG_MINER_BR_CONVERTER_H_ + +#include "lib/thread/ob_multi_fixed_queue_thread.h" +#include "ob_log_miner_error_handler.h" + +namespace oceanbase +{ +namespace oblogminer +{ +class ObLogMinerBR; +class ObLogMinerRecord; +class ILogMinerDataManager; +class ILogMinerResourceCollector; +class ILogMinerAnalysisWriter; + +class ILogMinerBRConverter { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int push(ObLogMinerBR *logminer_br) = 0; + virtual int get_total_task_count(int64_t &record_count) = 0; +}; + +typedef common::ObMQThread<1, ILogMinerBRConverter> BRConverterThreadPool; + +class ObLogMinerBRConverter: public ILogMinerBRConverter, public BRConverterThreadPool +{ +public: + static const int64_t BR_CONVERTER_THREAD_NUM; + static const int64_t BR_CONVERTER_QUEUE_SIZE; + static const int64_t PUSH_BR_TIMEOUT_TIME; +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int push(ObLogMinerBR *logminer_br); + virtual int get_total_task_count(int64_t &record_count); + +public: + ObLogMinerBRConverter(); + ~ObLogMinerBRConverter(); + int init(ILogMinerDataManager *data_manager, + ILogMinerAnalysisWriter *writer, + ILogMinerResourceCollector *resource_collector, + ILogMinerErrorHandler *err_handle_); + + virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag); + +private: + int generate_logminer_record_(ObLogMinerBR &br, ObLogMinerRecord &record); + +private: + bool is_inited_; + ILogMinerDataManager *data_manager_; + ILogMinerAnalysisWriter *writer_; + ILogMinerResourceCollector *resource_collector_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br_filter.cpp b/src/logservice/logminer/ob_log_miner_br_filter.cpp new file mode 100644 index 0000000000..584e6967ab --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br_filter.cpp @@ -0,0 +1,469 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_br_filter.h" +#include "ob_log_miner_br.h" +#include "ob_log_miner_br_producer.h" +#include "ob_log_miner_br_converter.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +//////////////////////////////////// ColumnBRFilterPlugin /////////////////////////////////// + +ColumnBRFilterPlugin::ColumnBRFilterPlugin(ObIAllocator *alloc): + alloc_(alloc), + multi_table_column_cond_(alloc_) + { } + +int ColumnBRFilterPlugin::init(const char *table_cond_str) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(multi_table_column_cond_.init(table_cond_str))) { + LOG_ERROR("multi_table_cond failed to init", K(table_cond_str), K(alloc_)); + } + + return ret; +} + +bool ColumnBRFilterPlugin::need_process_(const RecordType type) +{ + bool bret = false; + switch (type) { + case EINSERT: + case EUPDATE: + case EDELETE: + bret = true; + break; + default: + bret = false; + break; + } + return bret; +} + +bool ColumnBRFilterPlugin::is_table_match_(const ObLogMinerTableColumnCond& table_column_cond, + const char *db_name, + const char *table_name) +{ + return (table_column_cond.db_name_.is_empty() || 0 == strcmp(table_column_cond.db_name_.ptr(), db_name)) && + (table_column_cond.table_name_.is_empty() || 0 == strcmp(table_column_cond.table_name_.ptr(), table_name)); +} + +bool ColumnBRFilterPlugin::satisfy_column_cond_( + const ObLogMinerColumnVals &col_cond, + const binlogBuf *new_cols, + const int64_t new_col_cnt, + const binlogBuf *old_cols, + const int64_t old_col_cnt, + ITableMeta *tbl_meta) +{ + bool satisfy_new_col_cond_met = true; + bool satisfy_old_col_cond_met = true; + const int64_t col_cond_cnt = col_cond.count(); + ARRAY_FOREACH_X(col_cond, col_val_idx, col_cond_cnt, + (satisfy_old_col_cond_met || satisfy_new_col_cond_met)) { + const ObLogMinerColVal &col_val = col_cond[col_val_idx]; + int64_t column_idx = tbl_meta->getColIndex(col_val.col_.ptr()); + if (0 > column_idx) { + satisfy_new_col_cond_met = false; + satisfy_old_col_cond_met = false; + LOG_TRACE("col in col_cond doesn't exist in the row", K(column_idx), K(col_val)); + } else { + const char *new_data_ptr = nullptr, *old_data_ptr = nullptr; + int64_t new_data_len = 0, old_data_len = 0; + if (column_idx < new_col_cnt) { + new_data_ptr = new_cols[column_idx].buf; + new_data_len = new_cols[column_idx].buf_used_size; + } else { + // -1 indicates new_data don't exist + new_data_len = -1; + } + if (column_idx < old_col_cnt) { + old_data_ptr = old_cols[column_idx].buf; + old_data_len = old_cols[column_idx].buf_used_size; + } else { + // -1 indicates old_data don't exist + old_data_len = -1; + } + + if (satisfy_new_col_cond_met) { + satisfy_new_col_cond_met = is_data_match_column_val_(new_data_ptr, new_data_len, col_val); + } + if (satisfy_old_col_cond_met) { + satisfy_old_col_cond_met = is_data_match_column_val_(old_data_ptr, old_data_len, col_val); + } + } + } + + return satisfy_old_col_cond_met || satisfy_new_col_cond_met; +} + +bool ColumnBRFilterPlugin::is_data_match_column_val_(const char *data_ptr, + const int64_t data_len, + const ObLogMinerColVal &col_val) +{ + bool bret = true; + if (data_len != col_val.val_.length()) { + bret = false; + LOG_TRACE("the length of new_data is not same as the length of new col_val", K(col_val), + K(data_ptr), K(data_len)); + } else if (nullptr == data_ptr) { + if (col_val.is_null_) { + bret = true; + LOG_TRACE("data_ptr and the val in col_val are both null", K(col_val), + K(data_ptr)); + } else { + bret = false; + } + } else { + if (0 != MEMCMP(data_ptr, col_val.val_.ptr(), min(data_len, col_val.val_.length()))) { + bret = false; + LOG_TRACE("new_data doesn't match", K(col_val), + K(data_ptr)); + } else { + bret = true; + LOG_TRACE("new_data match successfully", K(col_val), + K(data_ptr)); + } + } + return bret; +} + +int ColumnBRFilterPlugin::filter(ObLogMinerBR &br, bool &need_filter) +{ + int ret = OB_SUCCESS; + need_filter = false; + ICDCRecord *cdc_br = br.get_br(); + if (OB_ISNULL(cdc_br)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get a null cdc binlog record, unexpected", K(cdc_br)); + } else if (!need_process_(static_cast(cdc_br->recordType()))) { + need_filter = false; + LOG_TRACE("doesn't filter some record needn't to be processed"); + } else { + ITableMeta *tbl_meta = cdc_br->getTableMeta(); + if (OB_ISNULL(tbl_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null table meta when filter record", "recordType", cdc_br->recordType()); + } else { + const char *tenant_db_name = cdc_br->dbname(); + const char *tbl_name = cdc_br->tbname(); + unsigned int new_col_cnt = 0, old_col_cnt = 0; + binlogBuf *new_cols = cdc_br->newCols(new_col_cnt); + binlogBuf *old_cols = cdc_br->oldCols(old_col_cnt); + + // expect tenant_db_name as tenant_name.db_name + const char *db_name = nullptr; + if (OB_ISNULL(tenant_db_name) || OB_ISNULL(tbl_name)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("dbname or tblname is null", K(tenant_db_name), K(tbl_name)); + } else if (0 == multi_table_column_cond_.table_column_conds_.count()) { + need_filter = false; + } else if (OB_ISNULL(db_name = STRCHR(tenant_db_name, '.') + 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected tenant_db_name format", K(tenant_db_name), K(tbl_name)); + } else { + bool satisfy_one_cond = false; + + DbAndTableWrapper db_and_table_wrapper(db_name, tbl_name); + ObLogMinerTableColumnCond *table_column_cond = nullptr; + if (OB_FAIL(multi_table_column_cond_.table_column_conds_.get(db_and_table_wrapper, table_column_cond))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_ERROR("get table column cond failed", K(db_and_table_wrapper)); + } + } else if (nullptr != table_column_cond) { + const int64_t column_conds_cnt = table_column_cond->column_conds_.count(); + ARRAY_FOREACH_X(table_column_cond->column_conds_, col_cond_idx, column_conds_cnt, + OB_SUCC(ret) && ! satisfy_one_cond) { + const ObLogMinerColumnVals &col_cond = table_column_cond->column_conds_[col_cond_idx]; + satisfy_one_cond = satisfy_column_cond_(col_cond, new_cols, + new_col_cnt, old_cols, old_col_cnt, tbl_meta); + } // ARRAY_FOREACH column_conds + + if (table_column_cond->column_conds_.empty()) { + satisfy_one_cond = true; + } + } + need_filter = !satisfy_one_cond; + } // else get legal arguments + } + } + + return ret; +} + +void ColumnBRFilterPlugin::destroy() +{ + multi_table_column_cond_.destroy(); +} + +//////////////////////////////////// OperationFilterPlugin /////////////////////////////////// +OperationBRFilterPlugin::OperationBRFilterPlugin(): + op_cond_() { } + +int OperationBRFilterPlugin::init(const char *op_cond_str) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(op_cond_.init(op_cond_str))) { + LOG_ERROR("op_cond init failed", K(op_cond_str)); + } + return ret; +} + +int OperationBRFilterPlugin::filter(ObLogMinerBR &br, bool &need_filter) +{ + int ret = OB_SUCCESS; + ICDCRecord *cdc_br = br.get_br(); + need_filter = false; + if (OB_ISNULL(cdc_br)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("operation filter get a null cdc br", K(cdc_br)); + } else { + need_filter = ! op_cond_.is_record_type_match(br.get_record_type()); + } + return ret; +} + +void OperationBRFilterPlugin::destroy() +{ + op_cond_.reset(); +} + +//////////////////////////////////// BRFilter /////////////////////////////////// + +const int64_t ObLogMinerBRFilter::BR_FILTER_THREAD_NUM = 1L; +const int64_t ObLogMinerBRFilter::BR_FILTER_QUEUE_SIZE = 100000L; + +ObLogMinerBRFilter::ObLogMinerBRFilter(): + is_inited_(false), + plugin_allocator("FilterPlugin"), + filter_pipeline_(), + data_manager_(nullptr), + resource_collector_(nullptr), + br_converter_(nullptr), + err_handle_(nullptr) { } + +ObLogMinerBRFilter::~ObLogMinerBRFilter() +{ + destroy(); +} + +int ObLogMinerBRFilter::init(const char *table_column_cond, + const char *op_cond, + ILogMinerDataManager *data_manager, + ILogMinerResourceCollector *resource_collector, + ILogMinerBRConverter *br_converter, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + ColumnBRFilterPlugin *column_plugin = nullptr; + OperationBRFilterPlugin *op_plugin = nullptr; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("oblogminer br_filter is already inited, no need to init again", K(is_inited_), + K(table_column_cond), K(op_cond), K(data_manager), K(resource_collector), K(br_converter)); + } else if (OB_FAIL(BRFilterThreadPool::init(BR_FILTER_THREAD_NUM, BR_FILTER_QUEUE_SIZE))) { + LOG_ERROR("BRFilterThreadPool failed to init"); + } else if (OB_ISNULL(column_plugin = + static_cast(plugin_allocator.alloc(sizeof(ColumnBRFilterPlugin))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for table filter plugin failed", K(column_plugin)); + } else if (OB_ISNULL(op_plugin = + static_cast(plugin_allocator.alloc(sizeof(OperationBRFilterPlugin))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for operation filter plugin failed", K(op_plugin)); + } else { + column_plugin = new (column_plugin) ColumnBRFilterPlugin(&plugin_allocator); + op_plugin = new (op_plugin) OperationBRFilterPlugin(); + if (OB_FAIL(column_plugin->init(table_column_cond))) { + LOG_ERROR("table plugin init failed", K(table_column_cond)); + } else if (OB_FAIL(op_plugin->init(op_cond))) { + LOG_ERROR("op plugin init failed", K(op_cond)); + } else if (OB_FAIL(filter_pipeline_.push_back(op_plugin))) { + LOG_ERROR("filter pipeline push back op_plugin failed", K(op_plugin), K(filter_pipeline_)); + } else if (OB_FAIL(filter_pipeline_.push_back(column_plugin))) { + LOG_ERROR("filter pipeline push back column_plugin failed", K(column_plugin), K(filter_pipeline_)); + } else { + data_manager_ = data_manager; + resource_collector_ = resource_collector; + br_converter_ = br_converter; + err_handle_ = err_handle; + is_inited_ = true; + LOG_INFO("ObLogMinerBRFilter finished to init", K(table_column_cond), K(op_cond)); + LOGMINER_STDOUT_V("ObLogMinerBRFilter finished to init\n"); + } + } + return ret; +} + +int ObLogMinerBRFilter::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRFilter hasn't been inited yet", K(is_inited_)); + } else if (OB_FAIL(BRFilterThreadPool::start())) { + LOG_ERROR("BRFilterThreadPool failed to start"); + } else { + LOG_INFO("ObLogMinerBRFilter starts successfully"); + } + return ret; +} + +void ObLogMinerBRFilter::stop() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + BRFilterThreadPool::mark_stop_flag(); + LOG_INFO("ObLogMinerBRFilter stopped"); + } else { + ret = OB_NOT_INIT; + LOG_WARN("ObLogMinerBRFilter is not init or has been stopped"); + } +} + +void ObLogMinerBRFilter::wait() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + BRFilterThreadPool::stop(); + LOG_INFO("ObLogMinerBRFilter finished to wait"); + } else { + ret = OB_NOT_INIT; + LOG_WARN("ObLogMinerBRFilter is not init or has been stopped"); + } +} + +void ObLogMinerBRFilter::destroy() +{ + if (IS_INIT) { + filter_pipeline_.destroy(); + data_manager_ = nullptr; + resource_collector_ = nullptr; + br_converter_ = nullptr; + plugin_allocator.clear(); + err_handle_ = nullptr; + BRFilterThreadPool::destroy(); + is_inited_ = false; + LOG_INFO("ObLogMinerBRFilter destroyed"); + LOGMINER_STDOUT_V("ObLogMinerBRFilter destroyed\n"); + } +} + +int ObLogMinerBRFilter::push(ObLogMinerBR *logminer_br) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRFilter is not inited", K(this)); + } else if (OB_ISNULL(logminer_br)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("ObLogMiner get a null logminer br", K(logminer_br)); + } else { + LOG_TRACE("BRFilter start to process logminer_br", KPC(logminer_br)); + uint64_t hash_val = logminer_br->get_seq_no(); // hash_val always equal to 0 + constexpr int64_t PUSH_TIMEOUT_TIME = 1L * 1000 * 1000; // 1 sec + // TODO: caculate hash_val for multi-thread + RETRY_FUNC(is_stoped(), *(static_cast(this)), push, logminer_br, hash_val, PUSH_TIMEOUT_TIME); + + if (OB_FAIL(ret)) { + LOG_ERROR("push logminer br into logminer brfilter failed", K(logminer_br), K(hash_val)); + } + } + return ret; +} + +int ObLogMinerBRFilter::get_total_task_count(int64_t &task_count) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRFilter is not inited", K(this)); + } else if (OB_FAIL(BRFilterThreadPool::get_total_task_num(task_count))) { + LOG_ERROR("failed to get total task num"); + } + + return ret; +} + +int ObLogMinerBRFilter::handle(void *data, const int64_t thread_index, volatile bool &stop_flag) +{ + int ret = OB_SUCCESS; + ObLogMinerBR *logminer_br = static_cast(data); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRFilter is not inited", K(is_inited_), K(data)); + } else if (OB_ISNULL(logminer_br)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("BRfilter got a null logminer br", K(logminer_br)); + } else { + bool need_filter = false; + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(filter_br_(*logminer_br, need_filter))) { + LOG_ERROR("ObLogMinerBRFilter failed to filter br", K(logminer_br), K(need_filter)); + } else if (need_filter) { + logminer_br->mark_filtered(); + } else { + // succ + } + + if (OB_SUCC(ret)) { + if (logminer_br->is_filtered()) { + // the RecordType of filtered br must be EINSERT, EUPDATE or EDELETE + // records of the RecordType above don't impact the progress, i.e. analysis checkpoint. + LOG_TRACE("br has been filtered", KPC(logminer_br)); + if (OB_FAIL(resource_collector_->revert(logminer_br))) { + LOG_ERROR("failed to revert logminer br", KPC(logminer_br), K(resource_collector_)); + } + } else { + if (OB_FAIL(br_converter_->push(logminer_br))) { + LOG_ERROR("failed to push logminer br into br converter", K(br_converter_), KPC(logminer_br)); + } + } + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "ObLogMinerBRFilter exit unexpected\n"); + } + return ret; +} + +int ObLogMinerBRFilter::filter_br_(ObLogMinerBR &br, bool &need_filter) +{ + int ret = OB_SUCCESS; + const int64_t filter_pipeline_size = filter_pipeline_.count(); + need_filter = false; + ARRAY_FOREACH_X(filter_pipeline_, filter_idx, filter_pipeline_size, OB_SUCC(ret) && ! need_filter) { + IBRFilterPlugin *filter_plugin = filter_pipeline_.at(filter_idx); + if (OB_ISNULL(filter_plugin)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get a null filter plugin in filter pipeline", K(filter_plugin)); + } else if (OB_FAIL(filter_plugin->filter(br, need_filter))) { + LOG_ERROR("filter_plugin failed to filter br", K(filter_plugin), K(need_filter)); + } + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br_filter.h b/src/logservice/logminer/ob_log_miner_br_filter.h new file mode 100644 index 0000000000..834797c133 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br_filter.h @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_BR_FILTER_H_ +#define OCEANBASE_LOG_MINER_BR_FILTER_H_ + +#include "lib/allocator/ob_concurrent_fifo_allocator.h" +#include "lib/thread/ob_multi_fixed_queue_thread.h" +#include "libobcdc.h" +#include "ob_log_miner_filter_condition.h" +#include "ob_log_miner_resource_collector.h" +#include "lib/container/ob_se_array.h" +#include "ob_log_miner_error_handler.h" + +namespace oceanbase +{ +namespace oblogminer +{ +class ObLogMinerBR; +class ILogMinerBRProducer; +class ILogMinerBRConverter; +class ILogMinerDataManager; + +class IBRFilterPlugin +{ +public: + virtual int filter(ObLogMinerBR &br, bool &need_filter) = 0; + DECLARE_PURE_VIRTUAL_TO_STRING; +}; + +class ColumnBRFilterPlugin: public IBRFilterPlugin +{ +public: + virtual int filter(ObLogMinerBR &br, bool &need_filter) override; + TO_STRING_KV(K_(multi_table_column_cond)); +public: + ColumnBRFilterPlugin(ObIAllocator *alloc); + ~ColumnBRFilterPlugin() { destroy(); }; + int init(const char *table_cond_str); + void destroy(); + + +private: + bool need_process_(const RecordType type); + bool is_table_match_(const ObLogMinerTableColumnCond& table_cond, + const char *db_name, + const char *table_name); + bool satisfy_column_cond_(const ObLogMinerColumnVals &col_cond, + const binlogBuf *new_cols, + const int64_t new_col_cnt, + const binlogBuf *old_cols, + const int64_t old_col_cnt, + ITableMeta *tbl_meta); + bool is_data_match_column_val_(const char *data_ptr, + const int64_t data_len, + const ObLogMinerColVal &col_val); + +private: + ObIAllocator *alloc_; + ObLogMinerMultiTableColumnCond multi_table_column_cond_; +}; + +class OperationBRFilterPlugin: public IBRFilterPlugin +{ +public: + virtual int filter(ObLogMinerBR &br, bool &need_filter) override; + TO_STRING_KV(K_(op_cond)); +public: + OperationBRFilterPlugin(); + ~OperationBRFilterPlugin() { destroy(); } + int init(const char *op_cond_str); + void destroy(); + +private: + ObLogMinerOpCond op_cond_; +}; + +class ILogMinerBRFilter { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int push(ObLogMinerBR *logminer_br) = 0; + virtual int get_total_task_count(int64_t &task_count) = 0; +}; + +typedef ObMQThread<1, ILogMinerBRFilter> BRFilterThreadPool; + +class ObLogMinerBRFilter: public ILogMinerBRFilter, public BRFilterThreadPool +{ +public: + static const int64_t BR_FILTER_THREAD_NUM; + static const int64_t BR_FILTER_QUEUE_SIZE; +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int push(ObLogMinerBR *logminer_br); + virtual int get_total_task_count(int64_t &task_count); + +public: + virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag); + +public: + ObLogMinerBRFilter(); + ~ObLogMinerBRFilter(); + int init(const char *table_cond, + const char *op_cond, + ILogMinerDataManager *data_manager, + ILogMinerResourceCollector *resource_collector, + ILogMinerBRConverter *br_converter, + ILogMinerErrorHandler *err_handle); + +private: + int filter_br_(ObLogMinerBR &br, bool &need_filter); + +private: + bool is_inited_; + ObArenaAllocator plugin_allocator; + common::ObSEArray filter_pipeline_; + ILogMinerDataManager *data_manager_; + ILogMinerResourceCollector *resource_collector_; + ILogMinerBRConverter *br_converter_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br_producer.cpp b/src/logservice/logminer/ob_log_miner_br_producer.cpp new file mode 100644 index 0000000000..c17048d033 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br_producer.cpp @@ -0,0 +1,330 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_br_producer.h" +#include "ob_log_miner_br_filter.h" +#include "ob_log_miner_args.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +const int64_t ObLogMinerBRProducer::BR_PRODUCER_THREAD_NUM = 1L; +const int64_t ObLogMinerBRProducer::BR_PRODUCER_POOL_DEFAULT_NUM = 4L * 1024; + +class ObCdcConfigMapWrap { +public: + explicit ObCdcConfigMapWrap(const std::map &cdc_config): + cdc_config_(cdc_config) { } + int64_t to_string(char *buf, const int64_t buf_len) const + { + int64_t pos = 0; + FOREACH(it, cdc_config_) { + if (need_print_config(it->first)) { + databuff_print_kv(buf, buf_len, pos, it->first.c_str(), it->second.c_str()); + } + } + return pos; + } + + bool need_print_config(const std::string &config_key) const { + bool need_print = true; + if ((0 == config_key.compare("tenant_password")) + || (0 == config_key.compare("archive_dest"))) { + need_print = false; + } + + return need_print; + } + +private: + const std::map &cdc_config_; +}; + +ObLogMinerBRProducer::ObLogMinerBRProducer(): + ThreadPool(BR_PRODUCER_THREAD_NUM), + is_inited_(false), + is_dispatch_end_(false), + start_time_us_(OB_INVALID_TIMESTAMP), + end_time_us_(OB_INVALID_TIMESTAMP), + curr_trans_id_(), + cdc_factory_(), + cdc_instance_(nullptr), + br_filter_(nullptr), + err_handle_(nullptr) { } + +int ObLogMinerBRProducer::init(const AnalyzerArgs &args, + ILogMinerBRFilter *filter, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + std::map cdc_config; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("oblogminer binlogrecord producer init twice", K(is_inited_), K(args.start_time_us_)); + } else if (OB_FAIL(build_cdc_config_map_(args, cdc_config))) { + LOG_ERROR("build cdc config map failed", K(args)); + } else if (OB_ISNULL(cdc_instance_ = cdc_factory_.construct_obcdc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc cdc_instance failed", K(cdc_instance_)); + } else if (OB_ISNULL(br_filter_ = filter)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("filter for br producer is null, unexpected", K(br_filter_), K(filter)); + } else if (OB_FAIL(cdc_instance_->init_with_start_tstamp_usec(cdc_config, + args.start_time_us_))) { + LOG_ERROR("init cdc instance failed", K(args.start_time_us_), K(ObCdcConfigMapWrap(cdc_config))); + LOGMINER_STDOUT("init cdc instance failed\n"); + } else if (OB_FAIL(lib::ThreadPool::init())) { + LOG_ERROR("thread pool failed to init"); + } else { + is_inited_ = true; + is_dispatch_end_ = false; + err_handle_ = err_handle; + start_time_us_ = args.start_time_us_; + end_time_us_ = args.end_time_us_; + LOG_INFO("ObLogMinerBRProducer finished to init", K(ObCdcConfigMapWrap(cdc_config)), + K(args.start_time_us_), K(filter)); + LOGMINER_STDOUT_V("ObLogMinerBRProducer finished to init\n"); + } + return ret; +} + +void ObLogMinerBRProducer::run1() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerBRProducer not init", K(is_inited_), K(cdc_instance_)); + } else if (OB_ISNULL(cdc_instance_) || OB_ISNULL(br_filter_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cdc_instance is null in br producer", K(cdc_instance_), K(is_inited_), K(br_filter_)); + } else { + bool next_dispatch_end = false; + const int64_t TIMEOUT_US = 1000L * 1000; + while (! has_set_stop() && OB_SUCC(ret)) { + ICDCRecord *br = nullptr; + ObLogMinerBR *logminer_br = nullptr; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + int32_t major_version = 0; + lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID; + if (OB_FAIL(cdc_instance_->next_record(&br, major_version, tenant_id,TIMEOUT_US))) { + if (OB_TIMEOUT != ret && OB_IN_STOP_STATE != ret) { + LOG_ERROR("get record from cdc_instance failed", K(cdc_instance_)); + LOGMINER_STDOUT_V("get record from cdc_instance failed\n"); + } + } else if (OB_ISNULL(br)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get a null ICDCRecord, unexpected", K(br), K(major_version), K(tenant_id)); + } else if (OB_INVALID_TENANT_ID == tenant_id) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get an invaild tenant_id", K(br), K(tenant_id), K(major_version)); + } else if (OB_FAIL(get_tenant_compat_mode_(static_cast(br->recordType()), + tenant_id, compat_mode))) { + LOG_ERROR("get tenant compat mode failed", K(tenant_id), K(compat_mode)); + } else if (lib::Worker::CompatMode::INVALID == compat_mode) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get an invalid compat mode", K(compat_mode), K(tenant_id)); + } else if (OB_FAIL(set_current_trans_id_(br))) { + LOG_ERROR("failed to set current trans_id", KP(br)); + } else if (OB_ISNULL(logminer_br = reinterpret_cast( + ob_malloc(sizeof(ObLogMinerBR), "LogMnrBR")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc logminer_br failed", K(logminer_br)); + } else { + logminer_br = new(logminer_br) ObLogMinerBR(); + if (OB_FAIL(logminer_br->init(cdc_instance_, br, compat_mode, + curr_trans_id_, tenant_id, major_version))) { + LOG_ERROR("logminer br failed to init", K(br), K(tenant_id), K(major_version)); + } else if (end_time_us_ < logminer_br->get_commit_scn().convert_to_ts()) { + // don't let any logminer_br whose commit_ts is larger than end_time_us go through + if (is_dispatch_end_) { + LOG_TRACE("destroy any br whose commit_ts larger than end_time", "commit_ts_us", + logminer_br->get_commit_scn(), K(end_time_us_)); + logminer_br->destroy(); + ob_free(logminer_br); + } else if (is_trans_end_record_type(logminer_br->get_record_type())) { + next_dispatch_end = true; + } else { + // still dispatch + } + } + } + + if (OB_SUCC(ret) && ! is_dispatch_end_) { + if (OB_FAIL(br_filter_->push(logminer_br))) { + LOG_ERROR("push logminer_br into br_filter failed", K(logminer_br), K(br_filter_)); + } else { + is_dispatch_end_ = next_dispatch_end; + } + } + + if (OB_TIMEOUT == ret) { + ret = OB_SUCCESS; + } else if (OB_IN_STOP_STATE == ret) { + LOG_INFO("cdc instance is stopped, br producer thread exits", K(cdc_instance_)); + } + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "ObLogMinerBRProducer exit unexpected\n"); + } +} + +int ObLogMinerBRProducer::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("br producer is not inited", K(cdc_instance_), K(br_filter_)); + } else if (OB_FAIL(cdc_instance_->launch())) { + LOG_ERROR("cdc instance failed to start"); + } else if (OB_FAIL(lib::ThreadPool::start())) { + LOG_ERROR("thread pool for br producer failed to start"); + } else { + LOG_INFO("ObLogMinerBRProducer starts"); + } + + return ret; +} + +void ObLogMinerBRProducer::stop() +{ + lib::ThreadPool::stop(); + LOG_INFO("ObLogMinerBRProducer stopped"); +} + +void ObLogMinerBRProducer::wait() +{ + lib::ThreadPool::wait(); + cdc_instance_->stop(); + LOG_INFO("ObLogMinerBRProducer finished to wait"); +} + +void ObLogMinerBRProducer::destroy() +{ + if (IS_INIT) { + ThreadPool::destroy(); + cdc_factory_.deconstruct(cdc_instance_); + cdc_instance_ = nullptr; + br_filter_ = nullptr; + err_handle_ = nullptr; + curr_trans_id_.reset(); + is_inited_ = false; + LOG_INFO("ObLogMinerBRProducer destroyed"); + LOGMINER_STDOUT_V("ObLogMinerBRProducer destroyed\n"); + } +} + +int ObLogMinerBRProducer::build_cdc_config_map_(const AnalyzerArgs &args, + std::map &cdc_config) +{ + int ret = OB_SUCCESS; + const char *REFRESH_MODE = "meta_data_refresh_mode"; + const char *FETCHING_MODE = "fetching_log_mode"; + const char *TENANT_ENDPOINT = "tenant_endpoint"; + const char *TENANT_USER = "tenant_user"; + const char *TENANT_PASSWORD = "tenant_password"; + const char *ARCHIVE_DEST = "archive_dest"; + const char *LOG_LEVEL = "log_level"; + const char *MEMORY_LIMIT = "memory_limit"; + const char *REDO_DISPATCH_LIMIT = "redo_dispatcher_memory_limit"; + const char *SQL_OUTPUT_ORDER = "enable_output_trans_order_by_sql_operation"; + const char *TB_WHITE_LIST = "tb_white_list"; + + try { + cdc_config[LOG_LEVEL] = args.log_level_; + cdc_config[REFRESH_MODE] = "data_dict"; + cdc_config[MEMORY_LIMIT] = "10G"; + cdc_config[REDO_DISPATCH_LIMIT] = "128M"; + cdc_config[SQL_OUTPUT_ORDER] = "1"; + cdc_config[TB_WHITE_LIST] = args.table_list_; + if (nullptr != args.archive_dest_) { + cdc_config[FETCHING_MODE] = "direct"; + cdc_config[ARCHIVE_DEST] = args.archive_dest_; + } else { + cdc_config[TENANT_ENDPOINT] = args.cluster_addr_; + cdc_config[TENANT_USER] = args.user_name_; + cdc_config[TENANT_PASSWORD] = args.password_; + } + } catch (std::exception &e) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("build cdc config map failed, exception occurs", "exception_info", e.what()); + } + + return ret; +} + +int ObLogMinerBRProducer::get_tenant_compat_mode_(const RecordType type, + const uint64_t tenant_id, + lib::Worker::CompatMode &mode) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("br producer has not been initialized yet", K(is_inited_), K(cdc_instance_)); + } else { + libobcdc::ObLogInstance *oblog_instance = static_cast(cdc_instance_); + if (HEARTBEAT == type) { + // in OBCDC, tenant_id is invalid for the HEARTBEAT type, + // and LogMiner won't deal with CompatMode for HEARTBEAT Record. + mode = lib::Worker::CompatMode::MYSQL; + } else if (OB_FAIL(oblog_instance->get_tenant_compat_mode(tenant_id, mode))) { + LOG_ERROR("get tenant compat mode failed", K(oblog_instance), K(tenant_id)); + } + } + return ret; +} + +int ObLogMinerBRProducer::set_current_trans_id_(ICDCRecord *cdc_rec) +{ + int ret = OB_SUCCESS; + ObString cdc_trans_id_str; + uint filter_val_cnt = 0; + BinlogRecordImpl *br_impl = static_cast(cdc_rec); + const binlogBuf *vals = br_impl->filterValues(filter_val_cnt); + const RecordType type = static_cast(cdc_rec->recordType()); + if (EBEGIN == type) { + if (OB_ISNULL(vals) || filter_val_cnt <= 2) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("trans_id values is null for begin record", KP(vals), K(filter_val_cnt)); + } else if (filter_val_cnt > 2) { + int64_t data = 0, pos = 0; + ObString start_str; + cdc_trans_id_str.assign_ptr(vals[1].buf, vals[1].buf_used_size); + start_str = cdc_trans_id_str.split_on('_'); + if (OB_FAIL(parse_int(cdc_trans_id_str.ptr(), cdc_trans_id_str.length(), pos, data))) { + LOG_ERROR("failed to parse into for trans_id", K(start_str), K(cdc_trans_id_str)); + } else { + curr_trans_id_ = data; + LOG_TRACE("parse trans_id for EBEGIN", K(start_str), K(cdc_trans_id_str), K(data)); + } + } else { + LOG_TRACE("defensive branch, do not set trans_id for EBEGIN whose vals is NULL", KP(vals)); + } + } else if (HEARTBEAT == type || EDDL == type) { + curr_trans_id_.reset(); + LOG_TRACE("reset trans_id for type HEARTBEAT and EDDL", K(type)); + } else { + // do not process other types + LOG_TRACE("ignore set current trans_id for type", K(type)); + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_br_producer.h b/src/logservice/logminer/ob_log_miner_br_producer.h new file mode 100644 index 0000000000..b8ddf2fa32 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_br_producer.h @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_BR_PRODUCER_H_ +#define OCEANBASE_LOG_MINER_BR_PRODUCER_H_ + +#include "ob_log_miner_br.h" +#include "ob_log_instance.h" +#include "ob_log_miner_error_handler.h" +#include "ob_log_miner_args.h" +#include +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerBRFilter; + +class ILogMinerBRProducer +{ +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; +}; + +class ObLogMinerBRProducer :public ILogMinerBRProducer, public lib::ThreadPool +{ +public: + static const int64_t BR_PRODUCER_THREAD_NUM; + static const int64_t BR_PRODUCER_POOL_DEFAULT_NUM; +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + // need to be idempotent + virtual void destroy(); + +private: + virtual void run1() override; + +public: + ObLogMinerBRProducer(); + ~ObLogMinerBRProducer() + { destroy(); } + int init(const AnalyzerArgs &args, + ILogMinerBRFilter *filter, + ILogMinerErrorHandler *err_handle); + +private: + int build_cdc_config_map_(const AnalyzerArgs &args, + std::map &cdc_config); + int get_tenant_compat_mode_(const RecordType type, + const uint64_t tenant_id, + lib::Worker::CompatMode &mode); + + int set_current_trans_id_(ICDCRecord *cdc_rec); +private: + bool is_inited_; + bool is_dispatch_end_; + int64_t start_time_us_; + int64_t end_time_us_; + transaction::ObTransID curr_trans_id_; + libobcdc::ObCDCFactory cdc_factory_; + libobcdc::IObCDCInstance *cdc_instance_; + ILogMinerBRFilter *br_filter_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_data_manager.cpp b/src/logservice/logminer/ob_log_miner_data_manager.cpp new file mode 100644 index 0000000000..9d5c954524 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_data_manager.cpp @@ -0,0 +1,300 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + + + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_record_converter.h" +#include "ob_log_miner_batch_record.h" +#include "ob_log_miner_data_manager.h" +#include "ob_log_miner_record.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ +const int64_t ObLogMinerDataManager::THREAD_NUM = 1; + +ObLogMinerDataManager::ObLogMinerDataManager(): + lib::ThreadPool(THREAD_NUM), + is_inited_(false), + stat_(), + mode_(LogMinerMode::UNKNOWN), + format_(RecordFileFormat::INVALID), + logminer_record_alloc_(), + start_progress_(OB_INVALID_TIMESTAMP), + end_progress_(OB_INVALID_TIMESTAMP), + output_progress_(OB_INVALID_TIMESTAMP), + record_count_(0), + err_handle_(nullptr) { } + +ObLogMinerDataManager::~ObLogMinerDataManager() +{ + destroy(); +} + +int ObLogMinerDataManager::init(const LogMinerMode mode, + const RecordFileFormat format, + const int64_t start_progress, + const int64_t end_progress, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("ObLogMinerDataManager has been initialized", K(is_inited_)); + } else if (OB_FAIL(logminer_record_alloc_.init(OB_MALLOC_NORMAL_BLOCK_SIZE, "MinerRecAlloc", + OB_SERVER_TENANT_ID, LOGMINER_RECORD_LIMIT))) { + LOG_ERROR("allocator for logminer record failed to init"); + } else if (OB_FAIL(logminer_batch_record_alloc_.init(OB_MALLOC_NORMAL_BLOCK_SIZE, "BatchRecAlloc", + OB_SERVER_TENANT_ID, LOGMINER_RECORD_LIMIT))) { + LOG_ERROR("allocator for logminer batch record failed to init"); + } else if (OB_FAIL(lib::ThreadPool::init())) { + LOG_ERROR("ThreadPool for data manager failed to init"); + } else { + stat_.reset(); + mode_ = mode; + format_ = format; + start_progress_ = start_progress; + end_progress_ = end_progress; + err_handle_ = err_handle; + is_inited_ = true; + LOG_INFO("ObLogMinerDataManager finished to init", K(mode), K(format), K(start_progress), + K(end_progress)); + LOGMINER_STDOUT_V("ObLogMinerDataManager finished to init\n"); + } + return ret; +} + +int ObLogMinerDataManager::start() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("data manager hasn't been initialized", K(is_inited_)); + } else if (OB_FAIL(lib::ThreadPool::start())) { + LOG_ERROR("thread pool for data manager failed to start"); + } else { + LOG_INFO("ObLogMinerDataManager starts"); + } + + return ret; +} + +void ObLogMinerDataManager::stop() +{ + lib::ThreadPool::stop(); + LOG_INFO("ObLogMinerDataManager stopped"); +} + +void ObLogMinerDataManager::wait() +{ + lib::ThreadPool::wait(); + LOG_INFO("ObLogMinerDataManager finished to wait"); +} + +void ObLogMinerDataManager::destroy() +{ + if (IS_INIT) { + is_inited_ = false; + stat_.reset(); + mode_ = LogMinerMode::UNKNOWN; + format_ = RecordFileFormat::INVALID; + start_progress_ = OB_INVALID_TIMESTAMP; + end_progress_ = OB_INVALID_TIMESTAMP; + lib::ThreadPool::destroy(); + logminer_batch_record_alloc_.destroy(); + logminer_record_alloc_.destroy(); + err_handle_ = nullptr; + LOG_INFO("ObLogMinerDataManager destroyed"); + LOGMINER_STDOUT_V("ObLogMinerDataManager destroyed\n"); + } +} + +int ObLogMinerDataManager::get_logminer_record(ObLogMinerRecord *&logminer_rec) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("get logminer record faild, data manager is not inited", K(is_inited_)); + } else { + if (OB_ISNULL(logminer_rec = op_alloc_args(ObLogMinerRecord, &logminer_record_alloc_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("logminer record pool alloc record failed", K(logminer_rec)); + } else { + ATOMIC_INC(&stat_.record_alloc_count_); + } + } + return ret; +} + +int ObLogMinerDataManager::release_log_miner_record(ObLogMinerRecord *logminer_rec) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("release logminer record faield, data manager is not inited", K(is_inited_)); + } else { + logminer_rec->destroy(); + op_free(logminer_rec); + ATOMIC_INC(&stat_.record_release_count_); + } + return ret; +} + +int ObLogMinerDataManager::get_logminer_batch_record(ObLogMinerBatchRecord *&logminer_batch_rec) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("get logminer record faield, data manager is not inited", K(is_inited_)); + } else { + ILogMinerRecordConverter *converter = ILogMinerRecordConverter::get_converter_instance(format_); + if (OB_ISNULL(converter)) { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("logminer doesn't support the output format", K(format_)); + } else if (OB_ISNULL(logminer_batch_rec = op_alloc_args(ObLogMinerBatchRecord, + &logminer_batch_record_alloc_, converter))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("logminer record pool alloc record failed", K(logminer_batch_rec)); + } else { + ATOMIC_INC(&stat_.batch_record_alloc_count_); + } + } + return ret; +} + +int ObLogMinerDataManager::release_logminer_batch_record(ObLogMinerBatchRecord *logminer_batch_rec) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("get logminer record faield, data manager is not inited", K(is_inited_)); + } else if (OB_ISNULL(logminer_batch_rec)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("cannot release a null logminer batch rec", K(logminer_batch_rec)); + } else { + logminer_batch_rec->reset(); + op_free(logminer_batch_rec); + ATOMIC_INC(&stat_.batch_record_release_count_); + } + return ret; +} + +int ObLogMinerDataManager::update_output_progress(const int64_t timestamp) +{ + int ret = OB_SUCCESS; + if (is_analysis_mode(mode_)) { + if (timestamp >= output_progress_) { + output_progress_ = timestamp; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("output progress rollbacked", K(timestamp), K(output_progress_), K(mode_)); + } + } else if (is_flashback_mode(mode_)) { + if (timestamp <= output_progress_) { + output_progress_ = timestamp; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("output progress rollbacked", K(timestamp), K(output_progress_), K(mode_)); + } + } + + if (OB_SUCC(ret)) { + if (reach_end_progress(timestamp)) { + if (OB_FAIL(LOGMINER_LOGGER.log_progress(record_count_, output_progress_, start_progress_, end_progress_))) { + LOG_WARN("log progress failed", K(record_count_), K(output_progress_), K(start_progress_), K(end_progress_)); + } + LOGMINER_STDOUT("\nObLogMinerAnalyzer completed process\n"); + err_handle_->handle_error(ret, "output progress reach end_progress"); + } + } + + return ret; +} + +int ObLogMinerDataManager::increase_record_count(int64_t count) { + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("data manager is not inited", K(is_inited_)); + } else if (count < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("increase negative count", K(count)); + } else { + record_count_ += count; + } + return ret; +} + +bool ObLogMinerDataManager::reach_end_progress(const int64_t progress) const +{ + bool bret = false; + if (OB_INVALID_TIMESTAMP == progress) { + bret = false; + LOG_WARN_RET(OB_ERR_UNEXPECTED, "get invalid progress when judging reach end progress", + K(progress), K(end_progress_)); + } else if (is_analysis_mode(mode_)) { + bret = progress > end_progress_; + } else if (is_flashback_mode(mode_)) { + bret = progress < end_progress_; + } + return bret; +} + +void ObLogMinerDataManager::run1() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("data manager not init"); + } else { + const int64_t STAT_INTERVAL = 1L * 1000 * 1000; // 1s + const int64_t LOOP_INTERVAL = 100L * 1000; // 100 ms + while (OB_SUCC(ret) && !has_set_stop()) { + if (REACH_TIME_INTERVAL(STAT_INTERVAL)) { + do_statistics_(); + } + ob_usleep(LOOP_INTERVAL); + } + } +} + +void ObLogMinerDataManager::do_statistics_() +{ + static LogMinerDataManagerStat last_stat; + int ret = OB_SUCCESS; + LogMinerDataManagerStat cur_stat = stat_; + LogMinerDataManagerStat delta; + delta.record_alloc_count_ = cur_stat.record_alloc_count_ - last_stat.record_alloc_count_; + delta.record_release_count_ = cur_stat.record_release_count_ - last_stat.record_release_count_; + delta.batch_record_alloc_count_ = cur_stat.batch_record_alloc_count_ - last_stat.batch_record_alloc_count_; + delta.batch_record_alloc_count_ = cur_stat.batch_record_alloc_count_ - last_stat.batch_record_alloc_count_; + LOG_INFO("[DATA_MANAGER] [STAT] [RECORD_COUNT]", K(cur_stat), K(last_stat), K(delta), + "active_record_cnt", cur_stat.record_alloc_count_ - cur_stat.record_release_count_, + "active_batch_record_cnt", cur_stat.batch_record_alloc_count_ - cur_stat.batch_record_release_count_); + LOG_INFO("[DATA_MANAGER] [STAT] [OUTPUT_PROGRESS]", K(output_progress_), K(start_progress_), K(end_progress_), + K(record_count_)); + // only print intermediate progress here. + if (output_progress_ != OB_INVALID_TIMESTAMP && !reach_end_progress(output_progress_)) { + if (OB_FAIL(LOGMINER_LOGGER.log_progress(record_count_, output_progress_, start_progress_, end_progress_))) { + LOG_WARN("log progress failed", K(record_count_), K(output_progress_), K(start_progress_), K(end_progress_)); + } + } + last_stat = cur_stat; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_data_manager.h b/src/logservice/logminer/ob_log_miner_data_manager.h new file mode 100644 index 0000000000..01b660d2ca --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_data_manager.h @@ -0,0 +1,144 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_DATA_MANAGER_H_ +#define OCEANBASE_LOG_MINER_DATA_MANAGER_H_ + +#include "lib/allocator/ob_concurrent_fifo_allocator.h" +#include "lib/objectpool/ob_small_obj_pool.h" +#include "lib/thread/threads.h" +#include "libobcdc.h" +#include "ob_log_miner_error_handler.h" +#include "ob_log_miner_mode.h" + +#include "ob_log_miner_file_manager.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerBR; +class ObLogMinerRecord; +class ObLogMinerBatchRecord; + +class ILogMinerDataManager { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int get_logminer_record(ObLogMinerRecord *&logminer_rec) = 0; + virtual int release_log_miner_record(ObLogMinerRecord *logminer_rec) = 0; + virtual int get_logminer_batch_record(ObLogMinerBatchRecord *&logminer_batch_rec) = 0; + virtual int release_logminer_batch_record(ObLogMinerBatchRecord *logminer_batch_rec) = 0; + virtual int update_output_progress(const int64_t timestamp) = 0; + virtual int64_t get_output_progress() const = 0; + virtual int64_t get_record_count() const = 0; + virtual int increase_record_count(int64_t count) = 0; + virtual bool reach_end_progress(const int64_t progress) const = 0; +}; + +class LogMinerDataManagerStat +{ +public: + LogMinerDataManagerStat() { reset(); } + ~LogMinerDataManagerStat() { reset(); } + + void reset() { + record_alloc_count_ = 0; + record_release_count_ = 0; + batch_record_alloc_count_ = 0; + batch_record_release_count_ = 0; + } + + // not atomic, just an estimate + LogMinerDataManagerStat &operator==(const LogMinerDataManagerStat &that) { + record_alloc_count_ = that.record_alloc_count_; + record_release_count_ = that.record_release_count_; + batch_record_alloc_count_ = that.batch_record_alloc_count_; + batch_record_release_count_ = that.batch_record_release_count_; + return *this; + } + + TO_STRING_KV( + K(record_alloc_count_), + K(record_release_count_), + K(batch_record_alloc_count_), + K(batch_record_release_count_) + ); + + int64_t record_alloc_count_ CACHE_ALIGNED; + int64_t record_release_count_ CACHE_ALIGNED; + int64_t batch_record_alloc_count_ CACHE_ALIGNED; + int64_t batch_record_release_count_ CACHE_ALIGNED; +}; + +class ObLogMinerDataManager: public ILogMinerDataManager, public lib::ThreadPool +{ + static const int64_t THREAD_NUM; + static const int64_t LOGMINER_RECORD_LIMIT = 2 * 1024L * 1024 * 1024; // 2G + static const int64_t BATCH_RECORD_LIMIT = 2 * 1024L * 1024 * 1024; // 2G +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int get_logminer_record(ObLogMinerRecord *&logminer_rec); + virtual int release_log_miner_record(ObLogMinerRecord *logminer_rec); + virtual int get_logminer_batch_record(ObLogMinerBatchRecord *&logminer_batch_rec); + virtual int release_logminer_batch_record(ObLogMinerBatchRecord *logminer_batch_rec); + virtual int update_output_progress(const int64_t timestamp); + virtual int64_t get_output_progress() const + { + return output_progress_; + } + virtual int64_t get_record_count() const + { + return record_count_; + } + virtual int increase_record_count(int64_t count); + virtual bool reach_end_progress(const int64_t progress) const; + +public: + virtual void run1(); + +public: + ObLogMinerDataManager(); + ~ObLogMinerDataManager(); + int init(const LogMinerMode mode, + const RecordFileFormat format, + const int64_t start_progress, + const int64_t end_progress, + ILogMinerErrorHandler *err_handle); + +private: + void do_statistics_(); + +private: + bool is_inited_; + LogMinerDataManagerStat stat_; + LogMinerMode mode_; + RecordFileFormat format_; + common::ObConcurrentFIFOAllocator logminer_record_alloc_; + common::ObConcurrentFIFOAllocator logminer_batch_record_alloc_; + int64_t start_progress_; + int64_t end_progress_; + int64_t output_progress_; + int64_t record_count_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_error_handler.h b/src/logservice/logminer/ob_log_miner_error_handler.h new file mode 100644 index 0000000000..35b1b8943d --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_error_handler.h @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_ERROR_HANDLER_H_ +#define OCEANBASE_LOG_MINER_ERROR_HANDLER_H_ + +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerErrorHandler +{ +public: + virtual void handle_error(int err_code, const char *fmt, ...) = 0; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_file_index.cpp b/src/logservice/logminer/ob_log_miner_file_index.cpp new file mode 100644 index 0000000000..623b762328 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_file_index.cpp @@ -0,0 +1,208 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_file_index.h" +#include "logservice/logminer/ob_log_miner_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +////////////////////////////// FileIndexItem ////////////////////////////// + +DEFINE_SERIALIZE(FileIndexItem) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf ,buf_len, pos, "%ld:%ld:%ld\n", + file_id_, range_.min_commit_ts_, range_.max_commit_ts_))) { + LOG_ERROR("failed to filled data into buf when serializing file index item", KPC(this)); + } + return ret; +} + +DEFINE_DESERIALIZE(FileIndexItem) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(parse_int(buf, data_len, pos, file_id_))) { + LOG_ERROR("failed to parse file_id", K(data_len), K(pos)); + } else if (OB_FAIL(expect_token(buf, data_len, pos, ":"))) { + LOG_ERROR("failed tot get token ':'", K(data_len), K(pos)); + } else if (OB_FAIL(parse_int(buf, data_len, pos, range_.min_commit_ts_))) { + LOG_ERROR("failed to parse min_commit_ts", K(data_len), K(pos)); + } else if (OB_FAIL(expect_token(buf, data_len, pos, ":"))) { + LOG_ERROR("failed tot get token ':'", K(data_len), K(pos)); + } else if (OB_FAIL(parse_int(buf, data_len, pos, range_.max_commit_ts_))) { + LOG_ERROR("failed to parse max_commit_ts", K(data_len), K(pos)); + } else if (OB_FAIL(expect_token(buf, data_len, pos, "\n"))) { + LOG_ERROR("failed tot get token '\\n'", K(data_len), K(pos)); + } + + return ret; +} + +DEFINE_GET_SERIALIZE_SIZE(FileIndexItem) +{ + int64_t size = 0; + const int64_t max_digit_len = 30; + char digit_buf[max_digit_len]; + size += snprintf(digit_buf, max_digit_len, "%ld", file_id_); + size += 1; + size += snprintf(digit_buf, max_digit_len, "%ld", range_.min_commit_ts_); + size += 1; + size += snprintf(digit_buf, max_digit_len, "%ld", range_.max_commit_ts_); + size += 1; + return size; +} + +////////////////////////////// FileIndex ////////////////////////////// + +int FileIndex::insert_index_item(const int64_t file_id, + const int64_t min_commit_ts, + const int64_t max_commit_ts) +{ + int ret = OB_SUCCESS; + FileIndexItem *item; + + if (OB_FAIL(alloc_index_item_(item))) { + LOG_ERROR("generate new index_item failed", K(item), K(file_id), K(min_commit_ts), K(max_commit_ts)); + } else if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null index item", K(item)); + } else { + item->reset(file_id, min_commit_ts, max_commit_ts); + if (OB_FAIL(index_array_.push_back(item))) { + LOG_ERROR("failed to push back item into index_array", KPC(item)); + } else { + index_file_len_ += item->get_serialize_size(); + } + } + + return ret; +} + +int FileIndex::insert_index_item(const FileIndexItem &item) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(insert_index_item(item.file_id_, item.range_.min_commit_ts_, item.range_.max_commit_ts_))) { + LOG_ERROR("failed to insert index_item", K(item)); + } + + return ret; +} + +int FileIndex::get_index_item(const int64_t file_id, FileIndexItem *&item) const +{ + int ret = OB_SUCCESS; + + if (file_id >= index_array_.count() || file_id < 0) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_ERROR("file id is out of range", K(file_id), "arr_count", index_array_.count()); + } else { + item = index_array_.at(file_id); + if (OB_ISNULL(item)) { + LOG_ERROR("get null item from index array", K(item), K(file_id), "arr_count", index_array_.count()); + } else if (item->file_id_ != file_id) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("the item is not wanted, unexpected", KPC(item), K(file_id)); + } + } + + return ret; +} + +int64_t FileIndex::get_index_file_len() const +{ + return index_file_len_; +} + +DEFINE_SERIALIZE(FileIndex) +{ + int ret = OB_SUCCESS; + ARRAY_FOREACH(index_array_, idx) + { + FileIndexItem *item = index_array_.at(idx); + if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null item when calculating serialized size", K(idx), K(item)); + } else if (OB_FAIL(item->serialize(buf, buf_len, pos))) { + LOG_ERROR("failed to serialize item into buf", K(buf_len), K(pos), K(idx)); + } + } + return ret; +} + +DEFINE_DESERIALIZE(FileIndex) +{ + int ret = OB_SUCCESS; + FileIndexItem item; + int64_t last_valid_pos = pos; + bool done = false; + while (OB_SUCC(ret) && !done) { + if (OB_FAIL(item.deserialize(buf, data_len, pos))) { + if (OB_INVALID_ARGUMENT != ret) { + LOG_WARN("failed to deserialize file_index_item", K(data_len), K(pos)); + } + ret = OB_SUCCESS; + pos = last_valid_pos; + done = true; + } else if (OB_FAIL(insert_index_item(item))) { + LOG_ERROR("failed to insert item into index array", K(item)); + } else { + last_valid_pos = pos; + if (pos >= data_len) { + done = true; + } + } + } + return ret; +} + +DEFINE_GET_SERIALIZE_SIZE(FileIndex) +{ + int64_t size = 0; + int ret = OB_SUCCESS; + ARRAY_FOREACH(index_array_, idx) + { + FileIndexItem *item = index_array_.at(idx); + if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null item when calculating serialized size", K(idx), K(item)); + } else { + size += item->get_serialize_size(); + } + } + return size; +} + + +int FileIndex::alloc_index_item_(FileIndexItem *&item) +{ + int ret = OB_SUCCESS; + + void *item_buf = alloc_.alloc(sizeof(FileIndexItem)); + if (OB_ISNULL(item_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("failed to allocate memory for file index item", K(item_buf)); + } else { + item = new(item_buf) FileIndexItem; + } + + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_file_index.h b/src/logservice/logminer/ob_log_miner_file_index.h new file mode 100644 index 0000000000..a35c4a55cc --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_file_index.h @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_FILE_INDEX_H_ +#define OCEANBASE_LOG_MINER_FILE_INDEX_H_ + +#include "lib/allocator/page_arena.h" +#include "lib/container/ob_se_array.h" +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" +#include "ob_log_miner_progress_range.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class FileIndexItem +{ +public: + FileIndexItem() { reset(); } + FileIndexItem(const int64_t file_id, const int64_t min_commit_ts, const int64_t max_commit_ts) + { reset(file_id, min_commit_ts, max_commit_ts); } + ~FileIndexItem() { reset(); } + + void reset() { + file_id_ = -1; + range_.reset(); + } + + void reset(const int64_t file_id, const int64_t min_commit_ts, const int64_t max_commit_ts) + { + file_id_ = file_id; + range_.min_commit_ts_ = min_commit_ts; + range_.max_commit_ts_ = max_commit_ts; + } + + bool operator==(const FileIndexItem &that) const + { + return file_id_ == that.file_id_ && range_ == that.range_; + } + + NEED_SERIALIZE_AND_DESERIALIZE; + + TO_STRING_KV( + K(file_id_), + K(range_) + ) + + int64_t file_id_; + ObLogMinerProgressRange range_; +}; + +class FileIndex +{ +public: + FileIndex(): + alloc_(), + index_file_len_(0), + index_array_() {} + ~FileIndex() { reset(); } + + void reset() { + index_file_len_ = 0; + index_array_.reset(); + alloc_.reset(); + } + + int insert_index_item(const int64_t file_id, const int64_t min_commit_ts, const int64_t max_commit_ts); + + int insert_index_item(const FileIndexItem &item); + + int get_index_item(const int64_t file_id, FileIndexItem *&item) const; + + int64_t get_index_file_len() const; + + NEED_SERIALIZE_AND_DESERIALIZE; + + TO_STRING_KV( + "file_num", index_array_.count() + ) +private: + int alloc_index_item_(FileIndexItem *&item); + +private: + ObArenaAllocator alloc_; + int64_t index_file_len_; + ObSEArray index_array_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_file_manager.cpp b/src/logservice/logminer/ob_log_miner_file_manager.cpp new file mode 100644 index 0000000000..b0841b1549 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_file_manager.cpp @@ -0,0 +1,661 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "share/backup/ob_backup_io_adapter.h" +#include "ob_log_miner_file_manager.h" +#include "ob_log_miner_batch_record.h" +#include "ob_log_miner_args.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +////////////////////////////// ObLogMinerFileManager ////////////////////////////// + +const char * ObLogMinerFileManager::META_PREFIX = "META/"; +const char * ObLogMinerFileManager::META_EXTENSION = "meta"; +const char * ObLogMinerFileManager::CSV_SUFFIX = "csv"; +const char * ObLogMinerFileManager::SQL_SUFFIX = "sql"; +const char * ObLogMinerFileManager::CONFIG_FNAME = "CONFIG"; +const char * ObLogMinerFileManager::CHECKPOINT_FNAME = "CHECKPOINT"; +const char * ObLogMinerFileManager::INDEX_FNAME = "COMMIT_INDEX"; +const char * ObLogMinerFileManager::EXITCODE_FNAME = "EXITCODE"; + +ObLogMinerFileManager::ObLogMinerFileManager(): + is_inited_(false), + output_dest_(), + mode_(FileMgrMode::INVALID), + format_(RecordFileFormat::INVALID), + last_write_ckpt_(), + meta_map_() { } + +ObLogMinerFileManager::~ObLogMinerFileManager() +{ + destroy(); +} + +int ObLogMinerFileManager::init(const char *path, + const RecordFileFormat format, + const FileMgrMode mode) +{ + int ret = OB_SUCCESS; + ObBackupIoAdapter adapter; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("ObLogMinerFileManager has been initialized", K(is_inited_), K(path), K(format), K(mode)); + } else if (FileMgrMode::RESTART == mode) { + // TODO: restart when interrupted + LOG_ERROR("unsupported mode", K(mode)); + } else if (RecordFileFormat::INVALID == format || FileMgrMode::INVALID == mode) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid record file format or filemgr mode", K(format), K(mode)); + } else if (OB_FAIL(output_dest_.set(path))) { + LOG_ERROR("output dest failed to set path", K(path)); + } else if (OB_FAIL(meta_map_.init("LogMinerFileIdx"))) { + LOG_ERROR("file index init failed"); + } else if (FileMgrMode::ANALYZE == mode && OB_FAIL(init_path_for_analyzer_())) { + LOG_ERROR("init path for analyzer failed", K(path), K(format), K(mode)); + // TODO: init for flashbacker and restart mode + } else { + last_write_ckpt_.reset(); + mode_ = mode; + format_ = format; + is_inited_ = true; + LOG_INFO("ObLogMinerFileManager finished to init", K(path), K(format), K(mode)); + LOGMINER_STDOUT_V("ObLogMinerFileManager finished to init\n"); + } + return ret; +} + +void ObLogMinerFileManager::destroy() +{ + if (IS_INIT) { + is_inited_ = false; + output_dest_.reset(); + mode_ = FileMgrMode::INVALID; + format_ = RecordFileFormat::INVALID; + last_write_ckpt_.reset(); + meta_map_.destroy(); + LOG_INFO("ObLogMinerFileManager destroyed"); + LOGMINER_STDOUT_V("ObLogMinerFileManager destroyed\n"); + } +} + +int ObLogMinerFileManager::append_records(const ObLogMinerBatchRecord &batch_record) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("file manager hasn't been initialized yet", K(is_inited_)); + } else { + ObLogMinerFileMeta meta; + const int64_t file_id = batch_record.get_file_id(); + const char *data = nullptr; + int64_t data_len = 0; + batch_record.get_data(data, data_len); + + // make sure to get meta here + if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) { + if (OB_ENTRY_NOT_EXIST == ret) { + if (OB_FAIL(create_data_file_(file_id))) { + LOG_ERROR("file manager failed to create file", K(file_id)); + } else if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) { + LOG_ERROR("failed to get file meta after create file", K(file_id)); + } + } else { + LOG_ERROR("get meta from meta index failed", K(meta)); + } + } + + // write file in append mode + if (OB_SUCC(ret)) { + if (0 < data_len && nullptr != data && OB_FAIL(append_data_file_(file_id, data, data_len))) { + LOG_ERROR("file manager write file failed", K(batch_record)); + } else { + const ObLogMinerProgressRange &range = batch_record.get_progress_range(); + meta.data_length_ += data_len; + if (OB_INVALID_TIMESTAMP == meta.range_.min_commit_ts_) { + meta.range_.min_commit_ts_ = range.min_commit_ts_; + } + + if (OB_INVALID_TIMESTAMP == meta.range_.min_commit_ts_ || + range.max_commit_ts_ >= meta.range_.max_commit_ts_) { + meta.range_.max_commit_ts_ = range.max_commit_ts_; + } else if (range.max_commit_ts_ < meta.range_.max_commit_ts_) { + // TODO: just print error, to findout commit version reverse + LOG_ERROR("range max_commit_ts rollbacked", K(meta), K(batch_record)); + } + + if (OB_FAIL(write_meta_(file_id, meta))) { + LOG_ERROR("write meta failed when appending batch_record", K(file_id), K(meta)); + } else if (OB_FAIL(meta_map_.insert_or_update(FileIdWrapper(file_id), meta))) { + LOG_ERROR("failed to update meta_map", K(file_id), K(meta)); + } + } + } + } + return ret; +} + +int ObLogMinerFileManager::write_config(const ObLogMinerArgs &args) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("file manager hasn't been initialized yet", K(is_inited_)); + } else { + char config_file_uri[OB_MAX_URI_LENGTH]; + const int64_t arg_size = args.get_serialize_size(); + ObArenaAllocator alloc; + char *config_file_data_buf = static_cast(alloc.alloc(arg_size + 1)); + int64_t pos = 0; + int64_t config_file_uri_len = 0; + if (OB_ISNULL(config_file_data_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for config_file_data failed", K(arg_size), K(args)); + } else if (OB_FAIL(config_file_uri_(config_file_uri, sizeof(config_file_uri), config_file_uri_len))) { + LOG_ERROR("failed to get config file uri"); + } else if (OB_FAIL(args.serialize(config_file_data_buf, arg_size + 1, pos))) { + LOG_ERROR("failed to serialize arg into buffer", K(arg_size), K(pos)); + } else { + ObBackupIoAdapter utils; + ObString config_file_uri_str(sizeof(config_file_uri), config_file_uri_len, config_file_uri); + if OB_FAIL(utils.write_single_file(config_file_uri_str, output_dest_.get_storage_info(), + config_file_data_buf, arg_size)) { + LOG_ERROR("failed to write config file to dest", K(config_file_uri_str), K(output_dest_), K(arg_size)); + } + } + + if (OB_NOT_NULL(config_file_data_buf)) { + alloc.free(config_file_data_buf); + } + } + return ret; +} + +int ObLogMinerFileManager::write_checkpoint(const ObLogMinerCheckpoint &ckpt) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("file manager hasn't been initialized yet", K(is_inited_)); + } else { + char checkpoint_file_uri[OB_MAX_URI_LENGTH]; + const int64_t ckpt_size = ckpt.get_serialize_size(); + ObArenaAllocator alloc; + char *ckpt_buf = static_cast(alloc.alloc(ckpt_size + 1)); + int64_t pos = 0; + int64_t checkpoint_file_uri_len = 0; + if (OB_ISNULL(ckpt_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocte memory for checkpoint failed", K(ckpt_size), K(ckpt)); + } else if (OB_FAIL(checkpoint_file_uri_(checkpoint_file_uri, + sizeof(checkpoint_file_uri), checkpoint_file_uri_len))) { + LOG_ERROR("get checkpoint file uri failed", K(checkpoint_file_uri_len)); + } else if (OB_FAIL(ckpt.serialize(ckpt_buf, ckpt_size+1, pos))) { + LOG_ERROR("ckpt failed to serialize to ckpt_buf", K(ckpt), K(ckpt_size), K(pos)); + } else { + // TODO: remove meta from meta_index according to max_file_id when writing checkpoint + ObBackupIoAdapter utils; + ObString ckpt_file_uri_str(sizeof(checkpoint_file_uri), checkpoint_file_uri_len, checkpoint_file_uri); + if (OB_FAIL(utils.write_single_file(ckpt_file_uri_str, output_dest_.get_storage_info(), + ckpt_buf, ckpt_size))) { + LOG_ERROR("failed to write config file to dest", K(checkpoint_file_uri), K(output_dest_), K(ckpt_size)); + } else if (OB_FAIL(update_last_write_ckpt_(ckpt))) { + LOG_ERROR("update last write ckpt failed", K(ckpt), K(last_write_ckpt_)); + } + } + + if (OB_NOT_NULL(ckpt_buf)) { + alloc.free(ckpt_buf); + } + } + + return ret; +} + +int ObLogMinerFileManager::read_file(const int64_t file_id, char *buf, const int64_t buf_len) +{ + int ret = OB_SUCCESS; + // TODO : read single file + return ret; +} + +int ObLogMinerFileManager::get_file_range(const int64_t min_timestamp_us, + const int64_t max_timestamp_us, + int64_t &min_file_id, + int64_t &max_file_id) +{ + int ret = OB_SUCCESS; + // TODO: flashback mode, get file range according to min_timestamp_us and max_timestamp_us + return ret; +} + +int ObLogMinerFileManager::read_config(ObLogMinerArgs &args) +{ + int ret = OB_SUCCESS; + // TODO: restart mode, read config to recover from exception + return ret; +} + +int ObLogMinerFileManager::read_checkpoint(ObLogMinerCheckpoint &ckpt) +{ + int ret = OB_SUCCESS; + // TODO: flashback/restart mode, get checkpoint + return ret; +} + +int ObLogMinerFileManager::config_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s", + output_dest_.get_root_path().ptr(), CONFIG_FNAME))) { + LOG_ERROR("fill config file uri into buffer failed", K(output_dest_), K(buf_len)); + } + return ret; +} + +int ObLogMinerFileManager::checkpoint_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s", + output_dest_.get_root_path().ptr(), CHECKPOINT_FNAME))) { + LOG_ERROR("fill checkpoint file uri into buffer failed", K(output_dest_), K(buf_len)); + } + return ret; +} + +int ObLogMinerFileManager::data_file_uri_( + const int64_t file_id, + char *buf, + const int64_t buf_len, + int64_t &length) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%ld.%s", + output_dest_.get_root_path().ptr(), file_id, data_file_extension_()))) { + LOG_ERROR("failed to get data file uri", K(length), K(file_id), K(buf_len)); + } + return ret; +} + +int ObLogMinerFileManager::meta_file_uri_( + const int64_t file_id, + char *buf, + const int64_t buf_len, + int64_t &length) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s%ld.%s", + output_dest_.get_root_path().ptr(), META_PREFIX, file_id, META_EXTENSION))) { + LOG_ERROR("failed to get meta file uri", K(length), K(file_id), K(buf_len)); + } + return ret; +} + +int ObLogMinerFileManager::index_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, length, "%s/%s", + output_dest_.get_root_path().ptr(), INDEX_FNAME))) { + LOG_ERROR("fill index file uri into buffer failed", K(output_dest_), K(buf_len)); + } + return ret; +} + +const char *ObLogMinerFileManager::data_file_extension_() const +{ + const char *suffix = ""; + switch(format_) { + case RecordFileFormat::CSV: { + suffix = "csv"; + break; + } + + case RecordFileFormat::REDO_ONLY: + case RecordFileFormat::UNDO_ONLY: { + suffix = "sql"; + break; + } + + case RecordFileFormat::JSON: { + suffix = "json"; + break; + } + + case RecordFileFormat::AVRO: { + suffix = "avro"; + break; + } + + case RecordFileFormat::PARQUET: { + suffix = "parquet"; + break; + } + + default: { + suffix = ""; + break; + } + } + return suffix; +} + +int ObLogMinerFileManager::create_data_file_(const int64_t file_id) +{ + int ret = OB_SUCCESS; + // create file, generate file header first. + int64_t pos = 0; + ObLogMinerFileMeta meta; + ObArenaAllocator alloc; + char *header_data = nullptr; + int64_t header_len = 0; + char data_file_uri[OB_MAX_URI_LENGTH]; + int64_t uri_len = 0; + + // create file meta, no need to write metainfo into storage + // but it's okay to write meta here + if (OB_FAIL(generate_data_file_header_(alloc, header_data, header_len))) { + LOG_ERROR("failed to generate file header", K(format_), K(header_len)); + } else if (OB_FAIL(data_file_uri_(file_id, data_file_uri, sizeof(data_file_uri), uri_len))) { + LOG_ERROR("failed to get data_file uri", K(file_id), K(uri_len)); + } else if (OB_FAIL(append_file_(data_file_uri, 0, header_data, header_len))) { + LOG_ERROR("failed to append file when creating file", K(data_file_uri), K(file_id), K(header_len)); + } else { + meta.range_.min_commit_ts_ = OB_INVALID_TIMESTAMP; + meta.range_.max_commit_ts_ = OB_INVALID_TIMESTAMP; + meta.data_length_ = header_len; + if (OB_FAIL(write_meta_(file_id, meta))) { + LOG_ERROR("failed to write meta when writing file", K(file_id), K(meta)); + } else if (OB_FAIL(meta_map_.insert(file_id, meta))) { + LOG_ERROR("failed to insert meta into meta_index", K(file_id), K(meta)); + } + } + + if (OB_NOT_NULL(header_data)) { + alloc.free(header_data); + } + + return ret; +} + +int ObLogMinerFileManager::generate_data_file_header_( + ObIAllocator &alloc, + char *&data, + int64_t &data_len) +{ + int ret = OB_SUCCESS; + switch (format_) { + case RecordFileFormat::CSV: { + #define MINER_SCHEMA_DEF(field, id, args...) \ + #field, + const char *header[] = { + #include "ob_log_miner_analyze_schema.h" + }; + #undef MINER_SCHEMA_DEF + int64_t header_len = 0, element_cnt = sizeof(header)/sizeof(const char*); + char *buf = nullptr; + int64_t pos = 0; + for (int i = 0; i < element_cnt; i++) { + header_len += strlen(header[i]) + 1; + } + if (OB_ISNULL(buf = static_cast(alloc.alloc(header_len + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for csv header failed", K(header_len)); + } else { + for (int i = 0; OB_SUCC(ret) && i < element_cnt; i++) { + char delimiter = i == element_cnt-1 ? '\n': ','; + if (OB_FAIL(databuff_printf(buf, header_len + 1, pos, "%s%c", header[i], delimiter))) { + LOG_ERROR("failed to fill header into buffer", K(i), K(header[i]), K(pos)); + } + } + + if (OB_SUCC(ret)) { + data = buf; + data_len = header_len; + } + } + break; + } + + case RecordFileFormat::REDO_ONLY: + case RecordFileFormat::UNDO_ONLY: + case RecordFileFormat::JSON: { + // do nothing; + break; + } + + // TODO: support other type + default: { + ret = OB_NOT_SUPPORTED; + break; + } + } + return ret; +} + +int ObLogMinerFileManager::append_data_file_(const int64_t file_id, const char *data, const int64_t data_len) +{ + int ret = OB_SUCCESS; + + ObBackupIoAdapter utils; + char data_file_uri[OB_MAX_URI_LENGTH]; + int64_t uri_len = 0; + ObIOFd fd; + ObIODevice *device_handle = nullptr; + ObLogMinerFileMeta meta; + int64_t write_size = 0; + + if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) { + LOG_ERROR("failed to get meta in meta_index when trying to append data to file", K(file_id), + K(data_len), K(meta)); + } else if (OB_FAIL(data_file_uri_(file_id, data_file_uri, sizeof(data_file_uri), uri_len))) { + LOG_ERROR("failed to get data_file uri", K(file_id), K(uri_len), K(output_dest_)); + } else if (OB_FAIL(append_file_(data_file_uri, meta.data_length_, data, data_len))) { + LOG_ERROR("failed to open device", K(file_id), K(output_dest_), K(data_file_uri)); + } + + return ret; +} + +int ObLogMinerFileManager::append_file_(const ObString &uri, + const int64_t offset, + const char *data, + const int64_t data_len) +{ + int ret = OB_SUCCESS; + + ObBackupIoAdapter utils; + ObIOFd fd; + ObIODevice *device_handle = nullptr; + int64_t write_size = 0; + if (nullptr == data || 0 == data_len) { + // do nothing + } else if (OB_FAIL(utils.open_with_access_type(device_handle, fd, output_dest_.get_storage_info(), + uri, common::OB_STORAGE_ACCESS_RANDOMWRITER))) { + LOG_ERROR("failed to open device", K(uri), K(output_dest_), K(uri)); + } else if (OB_FAIL(device_handle->pwrite(fd, offset, data_len, data, write_size))) { + LOG_ERROR("failed to write data into file", K(uri), K(output_dest_), + K(data_len), K(write_size)); + } else if (write_size != data_len) { + ret = OB_IO_ERROR; + LOG_WARN("write length not equal to data length", K(write_size), K(data_len)); + } + + if (nullptr != device_handle && fd.is_valid()) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(utils.close_device_and_fd(device_handle, fd))) { + LOG_ERROR("failed to close device and fd", K(uri), K(fd)); + } + } + return ret; +} + +int ObLogMinerFileManager::read_data_file_(const int64_t file_id, char *data, const int64_t data_len) +{ + int ret = OB_SUCCESS; + // TODO: read the whole file into the buf(data) + return ret; +} + +int ObLogMinerFileManager::write_data_file_(const int64_t file_id, const char *data, const int64_t data_len) +{ + int ret = OB_SUCCESS; + // TODO: write the whole file with data, used in restart mode. + return ret; +} + +int ObLogMinerFileManager::read_meta_(const int64_t file_id, ObLogMinerFileMeta &meta) +{ + int ret = OB_SUCCESS; + // TODO: read meta to read file. + return ret; +} + +int ObLogMinerFileManager::write_meta_(const int64_t file_id, const ObLogMinerFileMeta &meta) +{ + int ret = OB_SUCCESS; + char meta_uri[OB_MAX_URI_LENGTH]; + ObArenaAllocator alloc; + int64_t uri_len = 0; + ObBackupIoAdapter utils; + const int64_t buf_len = meta.get_serialize_size(); + int64_t pos = 0; + char *buf = static_cast(alloc.alloc(buf_len + 1)); + + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("failed to allocate memory to file meta data buffer", K(buf_len), K(buf)); + } else if (OB_FAIL(meta.serialize(buf, buf_len + 1, pos))) { + LOG_ERROR("failed to serialize meta into buffer", K(buf_len), K(meta)); + } else if (OB_FAIL(meta_file_uri_(file_id, meta_uri, sizeof(meta_uri), uri_len))) { + LOG_ERROR("failed to get meta_file uri", K(file_id), K(uri_len)); + } else if (OB_FAIL(utils.write_single_file(meta_uri, + output_dest_.get_storage_info(), buf, buf_len))) { + LOG_ERROR("failed to write meta file", K(meta), K(file_id), K(meta_uri), K(buf_len)); + } + + if (OB_NOT_NULL(buf)) { + alloc.free(buf); + } + + return ret; +} + +int ObLogMinerFileManager::write_index_(const int64_t file_id) +{ + int ret = OB_SUCCESS; + + ObLogMinerFileMeta meta; + char index_uri_cstr[OB_MAX_URI_LENGTH]; + int64_t index_uri_len = 0; + + if (OB_FAIL(meta_map_.get(FileIdWrapper(file_id), meta))) { + LOG_ERROR("get meta from meta_index failed", K(file_id), K(meta)); + } else if (OB_FAIL(index_file_uri_(index_uri_cstr, sizeof(index_uri_cstr), index_uri_len))) { + LOG_ERROR("failed to get index file uri", K(output_dest_), K(file_id), K(index_uri_len)); + } else { + const int64_t index_file_offset = file_index_.get_index_file_len(); + FileIndexItem item(file_id, meta.range_.min_commit_ts_, meta.range_.max_commit_ts_); + const int64_t index_data_len = item.get_serialize_size(); + ObArenaAllocator alloc; + int64_t pos = 0; + char *item_buf = static_cast(alloc.alloc(index_data_len + 1)); + if (OB_ISNULL(item_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for fileIndexItem failed", K(item_buf), K(index_data_len), K(item)); + } else if (OB_FAIL(item.serialize(item_buf, index_data_len + 1, pos))) { + LOG_ERROR("failed to serialize index item into item_buf", K(index_data_len), K(pos)); + } else if (OB_FAIL(append_file_(index_uri_cstr, index_file_offset, item_buf, index_data_len))) { + LOG_ERROR("failed to append index item data into storage system", K(index_uri_cstr), K(index_file_offset), + K(index_data_len)); + } else if (OB_FAIL(file_index_.insert_index_item(item))) { + LOG_ERROR("failed to insert index itme into file_index", K(file_index_), K(item)); + } + + } + + return ret; +} + +int ObLogMinerFileManager::update_last_write_ckpt_(const ObLogMinerCheckpoint &ckpt) +{ + int ret = OB_SUCCESS; + + const int64_t last_max_file_id = last_write_ckpt_.max_file_id_; + const int64_t cur_max_file_id = ckpt.max_file_id_; + + // traverse from last_max_file_id+1 to cur_max_file_id + for (int64_t i = last_max_file_id + 1; OB_SUCC(ret) && i <= cur_max_file_id; i++) { + if (OB_FAIL(write_index_(i))) { + LOG_ERROR("failed to write index for data file", "file_id", i, K(last_max_file_id), K(cur_max_file_id)); + } else if (OB_FAIL(meta_map_.erase(i))) { + LOG_ERROR("failed to erase meta from meta_index", "file_id", i, K(last_max_file_id), K(cur_max_file_id)); + } else { + } + } + + if (OB_SUCC(ret)) { + last_write_ckpt_ = ckpt; + } + + return ret; +} + +int ObLogMinerFileManager::init_path_for_analyzer_() +{ + int ret = OB_SUCCESS; + + ObBackupIoAdapter adapter; + char path_uri[OB_MAX_URI_LENGTH]; + bool is_exist = false; + + if (OB_FAIL(adapter.is_exist(output_dest_.get_root_path(), + output_dest_.get_storage_info(), is_exist))) { + LOG_ERROR("failed to check output_dest existence", K(output_dest_)); + } else if (! is_exist) { + if (OB_FAIL(adapter.mkdir(output_dest_.get_root_path(), output_dest_.get_storage_info()))) { + LOG_ERROR("failed to make output_dest", K(output_dest_)); + } + } + // oss don't create directory actually, so is_empty_directory() must be checked. + if(OB_SUCC(ret)) { + bool is_empty = false; + if (OB_FAIL(adapter.is_empty_directory(output_dest_.get_root_path(), + output_dest_.get_storage_info(), is_empty))) { + LOG_ERROR("failed to check is empty directory", K(output_dest_)); + } else if (!is_empty) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("init for analyzer but not empty directory", K(output_dest_), K(is_empty)); + LOGMINER_STDOUT("not empty output directory: %s\n", output_dest_.get_root_path().ptr()); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(databuff_printf(path_uri, sizeof(path_uri), "%s/%s", + output_dest_.get_root_path().ptr(), META_PREFIX))) { + LOG_ERROR("failed to fill path for meta entry", K(output_dest_), K(META_PREFIX)); + } else if (OB_FAIL(adapter.mkdir(path_uri, output_dest_.get_storage_info()))) { + LOG_ERROR("failed to mkdir for meta", K(path_uri), K(output_dest_)); + } + } + + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_file_manager.h b/src/logservice/logminer/ob_log_miner_file_manager.h new file mode 100644 index 0000000000..6025760409 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_file_manager.h @@ -0,0 +1,151 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_FILE_MANAGER_H_ +#define OCEANBASE_LOG_MINER_FILE_MANAGER_H_ + +#include "lib/hash/ob_linear_hash_map.h" +#include "share/backup/ob_backup_struct.h" +#include "ob_log_miner_file_index.h" +#include "ob_log_miner_progress_range.h" +#include "ob_log_miner_file_meta.h" +#include "ob_log_miner_analyzer_checkpoint.h" +#include "ob_log_miner_record_file_format.h" + + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerArgs; +class ObLogMinerBatchRecord; + +class ILogMinerFileManager +{ +public: + // write operation must satisfy the following order + // 1. write_config when logminer starts + // 2. write_checkpoint after append_record + // expect no parallel append to one file + virtual int append_records(const ObLogMinerBatchRecord &batch_record) = 0; + virtual int write_config(const ObLogMinerArgs &args) = 0; + virtual int write_checkpoint(const ObLogMinerCheckpoint &ckpt) = 0; + // read operation must satisfy the following order + // 1. read_config when logminer restarts + // 2. read_checkpoint before get_file_range to make sure get valid file range + // 3. read_checkpoint before read_file to make sure read valid file + virtual int read_file(const int64_t file_id, char *buf, const int64_t buf_len) = 0; + virtual int get_file_range(const int64_t min_timestamp_us, + const int64_t max_timestamp_us, + int64_t &min_file_id, + int64_t &max_file_id) = 0; + virtual int read_config(ObLogMinerArgs &args) = 0; + // read_checkpoint should be used only when logminer restarts + // there should not be concurrency issue with write_checkpoint() + virtual int read_checkpoint(ObLogMinerCheckpoint &ckpt) = 0; +}; + +struct FileIdWrapper { + FileIdWrapper(const int64_t file_id): fid_(file_id) {} + + int hash(uint64_t &val) const { + val = fid_; + return OB_SUCCESS; + } + + bool operator==(const FileIdWrapper &that) const { + return fid_ == that.fid_; + } + + int64_t fid_; +}; + +typedef ObLinearHashMap FileMetaMap; + +class ObLogMinerFileManager: public ILogMinerFileManager +{ +public: + enum class FileMgrMode { + INVALID = -1, + ANALYZE = 0, + FLASHBACK, + RESTART + }; + static const char *META_PREFIX; + static const char *META_EXTENSION; + static const char *CSV_SUFFIX; + static const char *SQL_SUFFIX; + static const char *CONFIG_FNAME; + static const char *CHECKPOINT_FNAME; + static const char *INDEX_FNAME; + static const char *EXITCODE_FNAME; +public: + virtual int append_records(const ObLogMinerBatchRecord &batch_record); + virtual int write_config(const ObLogMinerArgs &args); + virtual int write_checkpoint(const ObLogMinerCheckpoint &ckpt); + virtual int read_file(const int64_t file_id, char *buf, const int64_t buf_len); + virtual int get_file_range(const int64_t min_timestamp_us, + const int64_t max_timestamp_us, + int64_t &min_file_id, + int64_t &max_file_id); + virtual int read_config(ObLogMinerArgs &args); + virtual int read_checkpoint(ObLogMinerCheckpoint &ckpt); + +public: + ObLogMinerFileManager(); + virtual ~ObLogMinerFileManager(); + int init(const char *path, + const RecordFileFormat format, + const FileMgrMode mode); + void destroy(); + +private: + int config_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const; + int checkpoint_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const; + int data_file_uri_(const int64_t file_id, char *buf, const int64_t buf_len, int64_t &length) const; + int meta_file_uri_(const int64_t file_id, char *buf, const int64_t buf_len, int64_t &length) const; + int index_file_uri_(char *buf, const int64_t buf_len, int64_t &length) const; + const char *data_file_extension_() const; + + int create_data_file_(const int64_t file_id); + int generate_data_file_header_(ObIAllocator &alloc, char *&data, int64_t &data_len); + int append_data_file_(const int64_t file_id, const char *data, const int64_t data_len); + int append_file_(const ObString &uri, const int64_t offset, const char *data, const int64_t data_len); + int read_data_file_(const int64_t file_id, char *data, const int64_t data_len); + // overwrite the whole file with data + int write_data_file_(const int64_t file_id, const char *data, const int64_t data_len); + int read_meta_(const int64_t file_id, ObLogMinerFileMeta &meta); + int write_meta_(const int64_t file_id, const ObLogMinerFileMeta &meta); + // int file_exists_(const char *file_path) const; + + // only called by write_checkpoint + int write_index_(const int64_t file_id); + // only called by write_checkpoint + int update_last_write_ckpt_(const ObLogMinerCheckpoint &ckpt); + + int init_path_for_analyzer_(); + +private: + bool is_inited_; + share::ObBackupDest output_dest_; + FileMgrMode mode_; + RecordFileFormat format_; + ObLogMinerCheckpoint last_write_ckpt_; + FileMetaMap meta_map_; + FileIndex file_index_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_file_meta.cpp b/src/logservice/logminer/ob_log_miner_file_meta.cpp new file mode 100644 index 0000000000..ba0d5cc678 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_file_meta.cpp @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_file_meta.h" +#include "ob_log_miner_utils.h" +////////////////////////////// ObLogMinerFileMeta ////////////////////////////// + +namespace oceanbase +{ +namespace oblogminer +{ + +const char *ObLogMinerFileMeta::data_len_key = "DATA_LEN"; + +int ObLogMinerFileMeta::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(range_.serialize(buf, buf_len, pos))) { + LOG_ERROR("failed to serialize range into buf", K(range_), K(buf_len), K(pos)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n", + data_len_key, data_length_))) { + LOG_ERROR("failed to fill data_len into buf", K(buf_len), K(pos), K(data_length_)); + } + return ret; +} + +int ObLogMinerFileMeta::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(range_.deserialize(buf, data_len, pos))) { + LOG_ERROR("failed to deserialize range", K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(data_len_key, buf, data_len, pos, data_length_))) { + LOG_ERROR("failed to deserialize data_length", K(data_len_key), K(data_len), K(pos), K(data_length_)); + } + + return ret; +} + +int64_t ObLogMinerFileMeta::get_serialize_size() const +{ + int64_t size = 0; + const int64_t digit_max_len = 30; + char digit_str[digit_max_len]; + size += range_.get_serialize_size(); + size += strlen(data_len_key) + 2; + size += snprintf(digit_str, digit_max_len, "%ld", data_length_); + return size; +} + +} +} diff --git a/src/logservice/logminer/ob_log_miner_file_meta.h b/src/logservice/logminer/ob_log_miner_file_meta.h new file mode 100644 index 0000000000..31891c99aa --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_file_meta.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_FILE_META_H_ +#define OCEANBASE_LOG_MINER_FILE_META_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" +#include "ob_log_miner_progress_range.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +struct ObLogMinerFileMeta +{ +public: + + static const char *data_len_key; + + ObLogMinerFileMeta() {reset();} + ~ObLogMinerFileMeta() {reset();} + + void reset() { + range_.reset(); + data_length_ = 0; + } + + bool operator==(const ObLogMinerFileMeta &that) const + { + return range_ == that.range_ && data_length_ == that.data_length_; + } + + NEED_SERIALIZE_AND_DESERIALIZE; + + TO_STRING_KV( + K(range_), + K(data_length_) + ) + +public: + ObLogMinerProgressRange range_; + int64_t data_length_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_filter_condition.cpp b/src/logservice/logminer/ob_log_miner_filter_condition.cpp new file mode 100644 index 0000000000..5c3628e6a5 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_filter_condition.cpp @@ -0,0 +1,427 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_filter_condition.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +int ObLogMinerColVal::init(const ObString &column_name, + const ObString &value, + const bool is_null, + ObIAllocator *alloc) +{ + int ret = OB_SUCCESS; + char *value_buf = nullptr; + allocator_ = alloc; + if (OB_ISNULL(alloc)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null allocator, unexpected", K(alloc), K(column_name), K(value)); + } else if (OB_FAIL(col_.assign(column_name))) { + LOG_ERROR("assign column_name failed", K(column_name)); + } else { + if (is_null) { + is_null_ = true; + } else { + if (OB_ISNULL(value_buf = static_cast(alloc->alloc(value.length() + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory failed for value buf", K(value)); + } else { + MEMCPY(value_buf, value.ptr(), value.length()); + value_buf[value.length()] = '\0'; + val_.assign(value_buf, value.length()); + is_null_ = false; + } + } + } + return ret; +} + +void ObLogMinerColVal::destroy() +{ + col_.reset(); + if (nullptr != allocator_) { + allocator_->free(val_.ptr()); + } + allocator_ = nullptr; +} + +int DbAndTableWrapper::hash(uint64_t &val) const +{ + uint64_t hash_val = 0; + hash_val = common::murmurhash(db_name_.ptr(), db_name_.size(), hash_val); + hash_val = common::murmurhash(table_name_.ptr(), table_name_.size(), hash_val); + return OB_SUCCESS; + } + +const char *ObLogMinerTableColumnCond::DATABASE_KEY = "database_name"; +const char *ObLogMinerTableColumnCond::TABLE_KEY = "table_name"; +const char *ObLogMinerTableColumnCond::COLUMN_COND_KEY = "column_cond"; + +int ObLogMinerTableColumnCond::init(ObIAllocator *alloc, json::Object &obj) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(alloc)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("allocator fo oblogminer table cond is null", K(alloc)); + } else { + alloc_ = alloc; + } + DLIST_FOREACH(it, obj) { + if (OB_ISNULL(it)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get nulll pair in json, unexpected", K(it)); + } else if (0 == it->name_.case_compare(DATABASE_KEY)) { + if (OB_ISNULL(it->value_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get database key get null value", "key", it->name_, + "value", it->value_); + } else if (json::JT_STRING != it->value_->get_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get database key but value type is not expected", "key", it->name_, + "value", it->value_, "value_type", it->value_->get_type()); + } else if (! db_name_.is_empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get database key but db_name has been set already", K(db_name_), "key", it->name_, + "value", it->value_->get_string()); + } else if (OB_FAIL(db_name_.assign(it->value_->get_string()))) { + LOG_ERROR("assign db_name failed", K(db_name_), "value", it->value_->get_string()); + } else { + // succ + } + } else if (0 == it->name_.case_compare(TABLE_KEY)) { + if (OB_ISNULL(it->value_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get table key get null value", "key", it->name_, + "value", it->value_); + } else if (json::JT_STRING != it->value_->get_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get table key but value type is not expected", "key", it->name_, + "value", it->value_, "value_type", it->value_->get_type()); + } else if (! table_name_.is_empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get table key but table_name has been set already", K(table_name_), "key", it->name_, + "value", it->value_->get_string()); + } else if (OB_FAIL(table_name_.assign(it->value_->get_string()))) { + LOG_ERROR("assign table_name failed", K(table_name_), "value", it->value_->get_string()); + } else { + // succ + } + } else if (0 == it->name_.case_compare(COLUMN_COND_KEY)) { + if (OB_NOT_NULL(it->value_)) { + if (json::JT_ARRAY != it->value_->get_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get condtion key, expect value of array type, but got unexpected type", "key", it->name_, + "value_type", it->value_->get_type()); + } else { + DLIST_FOREACH(col_cond, it->value_->get_array()) { + if (OB_ISNULL(col_cond)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("column_cond key is specified, get null column cond, unexpetced", "key", it->name_, + K(col_cond)); + } else if (json::JT_OBJECT != col_cond->get_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get column cond, but its type is not as expected, object type is expected", + "column_cond_type", col_cond->get_type()); + } else if (OB_FAIL(add_column_cond_(col_cond->get_object()))) { + LOG_ERROR("add column condition faile", "col_cond", col_cond->get_object()); + } else { + // succ + } + } + } + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid key for table cond", "key", it->name_); + } + } + + if (table_name_.is_empty() || db_name_.is_empty() || column_conds_.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("It's unexpected to not specify column_conds or table_name or db_name", K(table_name_), + K(db_name_), K(column_conds_)); + } + return ret; +} + +int ObLogMinerTableColumnCond::init(ObIAllocator *alloc, + const ObString &db, + const ObString &tbl) +{ + int ret = OB_SUCCESS; + + alloc_ = alloc; + if (OB_FAIL(db_name_.assign(db))) { + LOG_ERROR("failed to assign db into db_name", K(db)); + } else if (OB_FAIL(table_name_.assign(tbl))) { + LOG_ERROR("failed to assign tbl into table_name", K(tbl)); + } + + return ret; +} + +int ObLogMinerTableColumnCond::add_column_cond_(json::Object &obj) +{ + int ret = OB_SUCCESS; + ObLogMinerColumnVals col_cond; + DLIST_FOREACH(col_val, obj) { + ObLogMinerColVal col_val_pair; + bool is_null_value = json::JT_NULL == col_val->value_->get_type(); + if (OB_ISNULL(col_val)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null col_val, unexpected for json parser", K(col_val)); + } else if (OB_ISNULL(col_val->value_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid column condition, whose value for column is null", "column_name", col_val->name_); + } else if (json::JT_STRING != col_val->value_->get_type() && !is_null_value) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get unexpected column value type", "column_name", col_val->name_, + "column_val_type", col_val->value_->get_type()); + } else if (OB_FAIL(col_val_pair.init(col_val->name_, + col_val->value_->get_string(), is_null_value, alloc_))) { + LOG_ERROR("column_name-column_value pair failed to init", "name", col_val->name_, + "value", col_val->value_->get_string()); + } else if (OB_FAIL(col_cond.push_back(col_val_pair))) { + LOG_ERROR("failed to push back col_val_pair into column_conds", K(col_val_pair)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(column_conds_.push_back(col_cond))) { + LOG_ERROR("failed to push back column_cond into column cond array", K(col_cond)); + } + } + + return ret; +} + +void ObLogMinerTableColumnCond::destroy() +{ + db_name_.reset(); + table_name_.reset(); + column_conds_.destroy(); + alloc_ = nullptr; +} + +uint64_t ObLogMinerTableColumnCond::hash() const +{ + uint64_t hash_val = 0; + hash_val = common::murmurhash(db_name_.ptr(), db_name_.size(), hash_val); + hash_val = common::murmurhash(table_name_.ptr(), table_name_.size(), hash_val); + return hash_val; +} + +int ObLogMinerTableColumnCond::hash(uint64_t &hash_val) const +{ + hash_val = hash(); + return OB_SUCCESS; +} + +int ObLogMinerMultiTableColumnCond::add_table_cond_(json::Object &obj) +{ + int ret = OB_SUCCESS; + ObLogMinerTableColumnCond *cond = + static_cast(alloc_->alloc(sizeof(ObLogMinerTableColumnCond))); + if (OB_ISNULL(cond)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for ObLogMinerTableColumnCond failed", K(cond)); + } else { + cond = new (cond) ObLogMinerTableColumnCond; + if (OB_FAIL(cond->init(alloc_, obj))) { + LOG_ERROR("table_cond init failed", KP(cond)); + } else if (OB_FAIL(table_column_conds_.insert( + DbAndTableWrapper(cond->db_name_.ptr(), cond->table_name_.ptr()), cond))) { + LOG_ERROR("failed to push back table cond into table_cond_array", K(cond)); + } + } + + return ret; +} + +int ObLogMinerMultiTableColumnCond::parse_json_cond_(const char *table_cond_str) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator(ObModIds::JSON_PARSER); + json::Parser parser; + json::Value *root = nullptr; + if (OB_ISNULL(table_cond_str)) { + LOG_INFO("get empty table cond str, don't filter any table", K(table_cond_str)); + } else if (OB_FAIL(parser.init(&allocator))) { + LOG_ERROR("init json parser for multi table cond failed"); + } else if (OB_FAIL(parser.parse(table_cond_str, strlen(table_cond_str), root))) { + LOG_ERROR("parse table_cond failed", K(table_cond_str), K(root)); + // rewrite the error code + ret = OB_INVALID_ARGUMENT; + } else { + if (json::JT_ARRAY == root->get_type()) { + DLIST_FOREACH(it, root->get_array()) { + if (OB_ISNULL(it)) { + ret = OB_ERR_UNEXPECTED; + } else if (json::JT_OBJECT == it->get_type()) { + if (OB_FAIL(add_table_cond_(it->get_object()))) { + LOG_ERROR("failed to add table cond", K(table_cond_str)); + } else { + // succ + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get unexpected table cond type in json", "type", it->get_type()); + } + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get unexpected root type in json, expected array", "type", root->get_type()); + } + } + + return ret; +} + +int ObLogMinerMultiTableColumnCond::init(const char *table_cond_str) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator(ObModIds::JSON_PARSER); + json::Parser parser; + json::Value *root = nullptr; + if (OB_ISNULL(table_cond_str) || 0 == strcmp(table_cond_str, "")) { + LOG_INFO("get empty table cond str, don't filter any table", K(table_cond_str)); + } else if (OB_FAIL(table_column_conds_.init("LogMnrColCond"))) { + LOG_ERROR("failed to init table_column_conds_", K(ret)); + } else if (OB_FAIL(parse_json_cond_(table_cond_str))) { + LOG_ERROR("failed to parse json cond", K(table_cond_str)); + } + + return ret; +} + +void ObLogMinerMultiTableColumnCond::destroy() +{ + table_column_conds_.destroy(); +} + +const char *ObLogMinerOpCond::INSERT_OP_STR = "insert"; +const char *ObLogMinerOpCond::UPDATE_OP_STR = "update"; +const char *ObLogMinerOpCond::DELETE_OP_STR = "delete"; +const char *ObLogMinerOpCond::OP_DELIMITER = "|"; + +int64_t ObLogMinerOpCond::record_type_to_bitcode(const RecordType type) { + int64_t ret = UNKNOWN_BITCODE; + switch (type) { + case EINSERT: { + ret = INSERT_BITCODE; + break; + } + + case EUPDATE: { + ret = UPDATE_BITCODE; + break; + } + + case EDELETE: { + ret = DELETE_BITCODE; + break; + } + + case EDDL: { + ret = DDL_BITCODE; + break; + } + + case HEARTBEAT: { + ret = HEARTBEAT_BITCODE; + break; + } + + case EBEGIN: { + ret = BEGIN_BITCODE; + break; + } + + case ECOMMIT: { + ret = COMMIT_BITCODE; + break; + } + + default: { + ret = UNKNOWN_BITCODE; + break; + } + } + return ret; +} + +int ObLogMinerOpCond::init(const char *op_cond) +{ + int ret = OB_SUCCESS; + char *saveptr = nullptr; + char *op = nullptr; + int64_t arg_len = op_cond == nullptr ? 0 : strlen(op_cond); + char *tmp_buffer = static_cast(ob_malloc(arg_len + 1, "LogMnrOpCond")); + if (OB_ISNULL(op_cond)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("ObLogMinerOpCond init with a null op_cond, unexpected", K(op_cond), K(arg_len)); + } else if (OB_ISNULL(tmp_buffer)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate temp buffer failed", K(tmp_buffer), K(arg_len), K(op_cond)); + } else { + MEMCPY(tmp_buffer, op_cond, arg_len); + tmp_buffer[arg_len] = '\0'; + // ddl, begin, commit, heartbeat need to be processed by default + op_cond_ |= (DDL_BITCODE | BEGIN_BITCODE | COMMIT_BITCODE | HEARTBEAT_BITCODE); + op = STRTOK_R(tmp_buffer, OP_DELIMITER, &saveptr); + while (nullptr != op && OB_SUCC(ret)) { + op = str_trim(op); + if (0 == strcasecmp(op, INSERT_OP_STR)) { + op_cond_ |= INSERT_BITCODE; + } else if (0 == strcasecmp(op, UPDATE_OP_STR)) { + op_cond_ |= UPDATE_BITCODE; + } else if (0 == strcasecmp(op, DELETE_OP_STR)) { + op_cond_ |= DELETE_BITCODE; + } else { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get unexpected operation", K(op_cond_), K(op_cond), K(op)); + LOGMINER_STDOUT("parse operations failed\n"); + break; + } + op = STRTOK_R(nullptr , OP_DELIMITER, &saveptr); + } + } + + if (OB_NOT_NULL(tmp_buffer)) { + ob_free(tmp_buffer); + } + return ret; +} + +bool ObLogMinerOpCond::is_record_type_match(const RecordType type) const +{ + bool bret = false; + const int64_t record_bitcode = record_type_to_bitcode(type); + if (record_bitcode & op_cond_) { + bret = true; + } + return bret; +} + +void ObLogMinerOpCond::reset() +{ + op_cond_ = 0; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_filter_condition.h b/src/logservice/logminer/ob_log_miner_filter_condition.h new file mode 100644 index 0000000000..44b025edab --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_filter_condition.h @@ -0,0 +1,188 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + */ + +#ifndef OCEANBASE_LOG_MINER_FILTER_CONDITION_H_ +#define OCEANBASE_LOG_MINER_FILTER_CONDITION_H_ + +#include "lib/json/ob_json.h" +#include "lib/string/ob_string_buffer.h" +#include "libobcdc.h" +#include "lib/container/ob_se_array.h" +#include "lib/hash/ob_linear_hash_map.h" +#include "lib/string/ob_fixed_length_string.h" +#include "ob_log_miner_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +struct ObLogMinerColVal +{ + ObLogMinerColVal(): + col_(), + val_(), + is_null_(false), + allocator_(nullptr) { } + ~ObLogMinerColVal() { destroy(); } + int init(const ObString &column_name, + const ObString &value, + const bool is_null, + ObIAllocator *alloc); + void destroy(); + + TO_STRING_KV(K(col_), K(val_)); + + ColumnName col_; + ObString val_; + bool is_null_; +private: + ObIAllocator *allocator_; +}; + +typedef ObSEArray ObLogMinerColumnVals; +struct DbAndTableWrapper +{ + DbAndTableWrapper(const char *db_name, const char *tbl_name): + db_name_(db_name), + table_name_(tbl_name) {} + + int hash(uint64_t &val) const; + + bool operator==(const DbAndTableWrapper &that) const { + return db_name_ == that.db_name_ && table_name_ == that.table_name_; + } + + TO_STRING_KV( + K_(db_name), + K_(table_name) + ); + + DbName db_name_; + TableName table_name_; +}; + +// Indicates column conditions of one table +struct ObLogMinerTableColumnCond +{ +public: + static const char *DATABASE_KEY; + static const char *TABLE_KEY; + static const char *COLUMN_COND_KEY; +public: + explicit ObLogMinerTableColumnCond(): + db_name_(), table_name_(), column_conds_(), alloc_(nullptr) { } + int init(ObIAllocator *alloc, json::Object &obj); + int init(ObIAllocator *alloc, const ObString &db, const ObString &tbl); + void destroy(); + + uint64_t hash() const; + int hash(uint64_t &hash_val) const; + + + TO_STRING_KV( + K_(db_name), + K_(table_name), + K_(column_conds) + ); + +private: + int add_column_cond_(json::Object &obj); + +public: + DbName db_name_; + TableName table_name_; + ObSEArray column_conds_; +private: + ObIAllocator *alloc_; +}; +// expect table cond is a json string with format below: +// [ +// { +// "database_name":"db1", +// "table_name":"tbl1", +// "column_cond":[ +// { +// "col1":"val1", +// "col2":"val2" +// }, +// { +// "col3":"val3", +// "col4":null +// } +// ] +// }, +// { +// "database_name":"db1", +// "table_name":"tbl2" +// } +// ] + +typedef ObLinearHashMap TableColumnCondMap; + +struct ObLogMinerMultiTableColumnCond +{ + explicit ObLogMinerMultiTableColumnCond(ObIAllocator *alloc): + table_column_conds_(), alloc_(alloc) { } + int init(const char *table_cond_str); + void destroy(); + + TO_STRING_KV("table_column_conds count ", table_column_conds_.count()); + +private: + int add_table_cond_(json::Object &obj); + + int parse_json_cond_(const char *table_cond_str); + +public: + TableColumnCondMap table_column_conds_; +private: + ObIAllocator *alloc_; +}; + + +struct ObLogMinerOpCond +{ +public: + // RecordType need to be processsed + static const int64_t INSERT_BITCODE = 1 << 0; + static const int64_t UPDATE_BITCODE = 1 << 1; + static const int64_t DELETE_BITCODE = 1 << 2; + static const int64_t DDL_BITCODE = 1 << 3; + static const int64_t HEARTBEAT_BITCODE = 1 << 4; + static const int64_t BEGIN_BITCODE = 1 << 5; + static const int64_t COMMIT_BITCODE = 1 << 6; + static const int64_t UNKNOWN_BITCODE = 1 << 63; + + static const char *INSERT_OP_STR; + static const char *UPDATE_OP_STR; + static const char *DELETE_OP_STR; + static const char *OP_DELIMITER; + + static int64_t record_type_to_bitcode(const RecordType type); + +public: + ObLogMinerOpCond(): + op_cond_(0) { } + int init(const char *op_cond); + void reset(); + bool is_record_type_match(const RecordType type) const; + TO_STRING_KV(K_(op_cond)); +public: + int64_t op_cond_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_flashback_reader.cpp b/src/logservice/logminer/ob_log_miner_flashback_reader.cpp new file mode 100644 index 0000000000..e981fc55c6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_flashback_reader.cpp @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_flashback_reader.h b/src/logservice/logminer/ob_log_miner_flashback_reader.h new file mode 100644 index 0000000000..5dcb0a0676 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_flashback_reader.h @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_FLASHBACK_READER_H_ +#define OCEANBASE_LOG_MINER_FLASHBACK_READER_H_ + +namespace oceanbase +{ +namespace oblogminer +{ +class ILogMinerFlashbackReader { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void destroy() = 0; +}; +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_flashback_writer.cpp b/src/logservice/logminer/ob_log_miner_flashback_writer.cpp new file mode 100644 index 0000000000..e981fc55c6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_flashback_writer.cpp @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_flashback_writer.h b/src/logservice/logminer/ob_log_miner_flashback_writer.h new file mode 100644 index 0000000000..ef57491f62 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_flashback_writer.h @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_FLASHBACK_WRITER_H_ +#define OCEANBASE_LOG_MINER_FLASHBACK_WRITER_H_ + +#include "lib/string/ob_sql_string.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerFlashbackWriter { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void destroy() = 0; + virtual int push(common::ObSqlString *sql) = 0; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_logger.cpp b/src/logservice/logminer/ob_log_miner_logger.cpp new file mode 100644 index 0000000000..cdb3361b38 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_logger.cpp @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include +#include +#include "ob_log_miner_logger.h" +#include "ob_log_miner_timezone_getter.h" +#include "lib/string/ob_string.h" +#include "lib/timezone/ob_time_convert.h" // ObTimeConverter + +namespace oceanbase +{ +namespace oblogminer +{ + +LogMinerLogger &LogMinerLogger::get_logminer_logger_instance() +{ + static LogMinerLogger logger_instance; + return logger_instance; +} +LogMinerLogger::LogMinerLogger(): + verbose_(false) { memset(pb_str_, '>', sizeof(pb_str_)); } + +void LogMinerLogger::log_stdout(const char *format, ...) +{ + va_list vl; + va_start(vl, format); + vfprintf(stdout, format, vl); + va_end(vl); +} + +void LogMinerLogger::log_stdout_v(const char *format, ...) +{ + if (verbose_) { + log_stdout(format); + } +} + +int LogMinerLogger::log_progress(int64_t record_num, int64_t current_ts, int64_t begin_ts, int64_t end_ts) +{ + int ret = OB_SUCCESS; + double percentage = double(current_ts - begin_ts) / double(end_ts - begin_ts); + double progress = 0; + int pb_width = 0; + int terminal_width = 0; + int lpad = 0; + int rpad = 0; + int64_t pos = 0; + const ObString nls_format; + char time_buf[128] = {0}; + char pb_buf[MAX_SCREEN_WIDTH] = {0}; + // current_ts may exceed end_ts + if (percentage > 1) { + percentage = 1; + } + progress = percentage * 100; + terminal_width = get_terminal_width(); + terminal_width = (terminal_width < MAX_SCREEN_WIDTH) ? terminal_width : MAX_SCREEN_WIDTH; + pb_width = terminal_width - FIXED_TERMINAL_WIDTH; + pb_width = (pb_width < MIN_PB_WIDTH) ? 0 : pb_width; + + if (0 < pb_width) { + lpad = (int)(percentage * pb_width); + rpad = pb_width - lpad; + sprintf(pb_buf, "[%.*s%*s]", lpad, pb_str_, rpad, ""); + } + if (OB_FAIL(ObTimeConverter::datetime_to_str(current_ts, &LOGMINER_TZ.get_tz_info(), + nls_format, 0, time_buf, sizeof(time_buf), pos))) { + LOG_WARN("datetime to string failed", K(current_ts), K(LOGMINER_TZ.get_tz_info())); + } else { + fprintf(stdout, "\r%s %s %5.1lf%%, written records: %-20jd", time_buf, pb_buf, + progress, record_num); + fflush(stdout); + } + return ret; +} + +int LogMinerLogger::get_terminal_width() +{ + int tmp_ret = OB_SUCCESS; + struct winsize terminal_size {}; + if (isatty(STDOUT_FILENO)) { + if (-1 == ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size)) { // On error, -1 is returned + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN_RET(tmp_ret, "get terminal width failed", K(errno), K(strerror(errno))); + } + } + // if get terminal width failed, return 0. + return tmp_ret == OB_SUCCESS ? terminal_size.ws_col : 0; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_logger.h b/src/logservice/logminer/ob_log_miner_logger.h new file mode 100644 index 0000000000..a28d69569d --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_logger.h @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_LOGGER_H_ +#define OCEANBASE_LOG_MINER_LOGGER_H_ + +namespace oceanbase +{ +namespace oblogminer +{ +#define LOGMINER_STDOUT(...) \ +::oceanbase::oblogminer::LogMinerLogger::get_logminer_logger_instance().log_stdout(__VA_ARGS__) +#define LOGMINER_STDOUT_V(...) \ +::oceanbase::oblogminer::LogMinerLogger::get_logminer_logger_instance().log_stdout_v(__VA_ARGS__) +#define LOGMINER_LOGGER \ +::oceanbase::oblogminer::LogMinerLogger::get_logminer_logger_instance() +class LogMinerLogger { +public: + LogMinerLogger(); + static LogMinerLogger &get_logminer_logger_instance(); + void set_verbose(bool verbose) { + verbose_ = verbose; + } + void log_stdout(const char *format, ...); + void log_stdout_v(const char *format, ...); + int log_progress(int64_t record_num, int64_t current_ts, int64_t begin_ts, int64_t end_ts); +private: + int get_terminal_width(); + +private: + static const int MIN_PB_WIDTH = 5; + // 68: 20->datatime, 2->[], 7->percentage, 19->", written records: ", 20->the length of INT64_MAX + static const int FIXED_TERMINAL_WIDTH = 68; + static const int MAX_SCREEN_WIDTH = 4096; + bool verbose_; + char pb_str_[MAX_SCREEN_WIDTH]; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_main.cpp b/src/logservice/logminer/ob_log_miner_main.cpp new file mode 100644 index 0000000000..8cd64de024 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_main.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner.h" +#include "ob_log_miner_args.h" +#include "ob_log_miner_logger.h" +#include "lib/oblog/ob_log.h" +#include "share/rc/ob_tenant_base.h" + +using namespace oceanbase::oblogminer; +using namespace oceanbase::common; +using namespace oceanbase::share; + +int main(int argc, char *argv[]) +{ + int ret = OB_SUCCESS; + ObLogMinerArgs args; + ObLogMiner logminer_instance; + // set OB_SERVER_TENANT_ID as oblogminer's MTL_ID. + // if not set, `ob_malloc` maybe generate error logs + // due to `OB_INVALID_TENANT_ID`. + ObTenantBase tenant_ctx(OB_SERVER_TENANT_ID); + ObTenantEnv::set_tenant(&tenant_ctx); + + OB_LOGGER.set_file_name(ObLogMinerArgs::LOGMINER_LOG_FILE, true, false); + OB_LOGGER.set_log_level("WDIAG"); + OB_LOGGER.set_enable_async_log(false); + + if (OB_FAIL(args.init(argc, argv))) { + LOG_ERROR("logminer get invalid arguments", K(argc), K(args)); + LOGMINER_STDOUT("logminer get invalid arguments, please check log[%s] for more detail\n", + ObLogMinerArgs::LOGMINER_LOG_FILE); + } else if (args.print_usage_) { + ObLogMinerCmdArgs::print_usage(argv[0]); + } else if (OB_FAIL(logminer_instance.init(args))) { + LOG_ERROR("logminer instance init failed", K(args)); + LOGMINER_STDOUT("logminer init failed, please check log[%s] for more detail\n", + ObLogMinerArgs::LOGMINER_LOG_FILE); + } else { + logminer_instance.run(); + } + + return ret; +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_mode.cpp b/src/logservice/logminer/ob_log_miner_mode.cpp new file mode 100644 index 0000000000..96b4aca237 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_mode.cpp @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_miner_mode.h" +#include + +namespace oceanbase +{ +namespace oblogminer +{ + +const char *logminer_mode_to_str(LogMinerMode mode) +{ + const char *mode_str = nullptr; + switch(mode) { + case LogMinerMode::UNKNOWN: { + mode_str = "UNKNOWN"; + break; + } + + case LogMinerMode::ANALYSIS: { + mode_str = "ANALYSIS"; + break; + } + + case LogMinerMode::FLASHBACK: { + mode_str = "FLASHBACK"; + break; + } + + case LogMinerMode::MAX_MODE: { + mode_str = "MAX_MODE"; + break; + } + + default: { + mode_str = "INVALID"; + break; + } + } + + return mode_str; +} + +LogMinerMode get_logminer_mode(const common::ObString &mode_str) +{ + LogMinerMode mode = LogMinerMode::UNKNOWN; + if (0 == mode_str.case_compare("analysis")) { + mode = LogMinerMode::ANALYSIS; + } else if (0 == mode_str.case_compare("flashback")) { + mode = LogMinerMode::FLASHBACK; + } else { + // return UNKNOWN mode + } + return mode; +} + +LogMinerMode get_logminer_mode(const char *mode_str) +{ + LogMinerMode mode = LogMinerMode::UNKNOWN; + + if (0 == strcasecmp(mode_str, "analysis")) { + mode = LogMinerMode::ANALYSIS; + } else if (0 == strcasecmp(mode_str, "flashback")) { + mode = LogMinerMode::FLASHBACK; + } else { + // return UNKNOWN mode + } + + return mode; +} + +bool is_logminer_mode_valid(const LogMinerMode mode) +{ + return mode > LogMinerMode::UNKNOWN && mode < LogMinerMode::MAX_MODE; +} + +bool is_analysis_mode(const LogMinerMode mode) +{ + return LogMinerMode::ANALYSIS == mode; +} + +bool is_flashback_mode(const LogMinerMode mode) +{ + return LogMinerMode::FLASHBACK == mode; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_mode.h b/src/logservice/logminer/ob_log_miner_mode.h new file mode 100644 index 0000000000..710cdbc02b --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_mode.h @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_MODE_H_ +#define OCEANBASE_LOG_MINER_MODE_H_ + +#include "lib/string/ob_string.h" + +namespace oceanbase +{ +namespace oblogminer +{ +enum class LogMinerMode { + UNKNOWN = 0, + + ANALYSIS = 1, + FLASHBACK = 2, + + MAX_MODE +}; + +const char *logminer_mode_to_str(LogMinerMode mode); +LogMinerMode get_logminer_mode(const common::ObString &mode_str); +LogMinerMode get_logminer_mode(const char *mode_str); +bool is_logminer_mode_valid(const LogMinerMode mode); +bool is_analysis_mode(const LogMinerMode mode); +bool is_flashback_mode(const LogMinerMode mode); + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_progress_range.cpp b/src/logservice/logminer/ob_log_miner_progress_range.cpp new file mode 100644 index 0000000000..9ba9fe7afb --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_progress_range.cpp @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_progress_range.h" +#include "ob_log_miner_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +////////////////////////////// ObLogMinerProgressRange ////////////////////////////// + +const char *ObLogMinerProgressRange::min_commit_ts_key = "MIN_COMMIT_TS"; +const char *ObLogMinerProgressRange::max_commit_ts_key = "MAX_COMMIT_TS"; + +int ObLogMinerProgressRange::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s=%ld\n%s=%ld\n", + min_commit_ts_key, min_commit_ts_, max_commit_ts_key, max_commit_ts_))) { + LOG_ERROR("failed to serialize progress range into buffer", K(buf_len), K(pos), KPC(this)); + } + return ret; +} + +int ObLogMinerProgressRange::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(parse_line(min_commit_ts_key, buf, data_len, pos, min_commit_ts_))) { + LOG_ERROR("parse line for min_commit_ts failed", K(min_commit_ts_key), K(data_len), K(pos)); + } else if (OB_FAIL(parse_line(max_commit_ts_key, buf, data_len, pos, max_commit_ts_))) { + LOG_ERROR("parse line for max_commit_ts failed", K(max_commit_ts_key), K(data_len), K(pos)); + } + return ret; +} + +int64_t ObLogMinerProgressRange::get_serialize_size() const +{ + int64_t size = 0; + const int64_t max_digit_size = 30; + char digit_str[max_digit_size]; + size += snprintf(digit_str, max_digit_size, "%ld", min_commit_ts_); + size += snprintf(digit_str, max_digit_size, "%ld", max_commit_ts_); + // min_commit_ts_key + max_commit_ts_key + '=' + '\n' + '=' + '\n' + size += strlen(min_commit_ts_key) + strlen(max_commit_ts_key) + 4; + return size; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_progress_range.h b/src/logservice/logminer/ob_log_miner_progress_range.h new file mode 100644 index 0000000000..fe304ca591 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_progress_range.h @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_PROGRESS_RANGE_H_ +#define OCEANBASE_LOG_MINER_PROGRESS_RANGE_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +struct ObLogMinerProgressRange +{ + static const char *min_commit_ts_key; + static const char *max_commit_ts_key; + + ObLogMinerProgressRange() + { reset(); } + + void reset() + { + min_commit_ts_ = OB_INVALID_TIMESTAMP; + max_commit_ts_ = OB_INVALID_TIMESTAMP; + } + + bool is_valid() const + { + return min_commit_ts_ != OB_INVALID_TIMESTAMP && max_commit_ts_ != OB_INVALID_TIMESTAMP; + } + + ObLogMinerProgressRange &operator=(const ObLogMinerProgressRange &that) + { + min_commit_ts_ = that.min_commit_ts_; + max_commit_ts_ = that.max_commit_ts_; + return *this; + } + + bool operator==(const ObLogMinerProgressRange &that) const + { + return min_commit_ts_ == that.min_commit_ts_ && max_commit_ts_ == that.max_commit_ts_; + } + + TO_STRING_KV( + K(min_commit_ts_), + K(max_commit_ts_) + ); + + NEED_SERIALIZE_AND_DESERIALIZE; + + int64_t min_commit_ts_; + int64_t max_commit_ts_; +}; + +} +} +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record.cpp b/src/logservice/logminer/ob_log_miner_record.cpp new file mode 100644 index 0000000000..87b41e2250 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record.cpp @@ -0,0 +1,1267 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "lib/utility/ob_fast_convert.h" +#include "ob_log_miner_record.h" +#include "ob_log_miner_br.h" +#include "ob_log_binlog_record.h" +#include "ob_log_miner_logger.h" + +#define APPEND_STMT(stmt, args...) \ + do { \ + if (OB_SUCC(ret)) { \ + if (OB_FAIL(stmt.append(args))) { \ + LOG_ERROR("failed to append args to stmt", K(stmt));\ + } \ + } \ + } while (0) + +// only used in ObLogMinerRecord +#define APPEND_ESCAPE_CHAR(stmt) \ + do { \ + if (OB_SUCC(ret)) { \ + if (OB_FAIL(build_escape_char_(stmt))) { \ + LOG_ERROR("failed to append escape char to stmt", K(stmt));\ + } \ + } \ + } while (0) + +namespace oceanbase +{ +namespace oblogminer +{ +const char *ObLogMinerRecord::ORACLE_ESCAPE_CHAR = "\""; +const char *ObLogMinerRecord::MYSQL_ESCAPE_CHAR = "`"; +const char *ObLogMinerRecord::ORA_GEO_PREFIX = "SRID="; +const char *ObLogMinerRecord::JSON_EQUAL = "JSON_EQUAL"; +const char *ObLogMinerRecord::LOB_COMPARE = "0=DBMS_LOB.COMPARE"; +const char *ObLogMinerRecord::ST_EQUALS = "ST_Equals"; + +ObLogMinerRecord::ObLogMinerRecord(): + ObLogMinerRecyclableTask(TaskType::LOGMINER_RECORD), + is_inited_(false), + is_filtered_(false), + alloc_(nullptr), + compat_mode_(lib::Worker::CompatMode::INVALID), + tenant_id_(OB_INVALID_TENANT_ID), + orig_cluster_id_(OB_INVALID_CLUSTER_ID), + tenant_name_(), + database_name_(), + table_name_(), + trans_id_(), + primary_keys_(), + unique_keys_(), + row_unique_id_(), + record_type_(EUNKNOWN), + commit_scn_(), + redo_stmt_(), + undo_stmt_() +{ +} + +ObLogMinerRecord::ObLogMinerRecord(ObIAllocator *alloc): + ObLogMinerRecord() +{ + set_allocator(alloc); +} + +ObLogMinerRecord::~ObLogMinerRecord() +{ + destroy(); +} + +int ObLogMinerRecord::init(ObLogMinerBR &logminer_br) +{ + int ret = OB_SUCCESS; + ICDCRecord *cdc_record = nullptr; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("ObLogMinerRecord has already been initialized", K(is_inited_)); + } else if (OB_ISNULL(alloc_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("allocator is null before init, unexpected", K(alloc_)); + } else if (OB_ISNULL(cdc_record = logminer_br.get_br())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get a null cdc record in logminer br", K(cdc_record)); + } else { + is_inited_ = true; + is_filtered_ = false; + record_type_ = logminer_br.get_record_type(); + compat_mode_ = logminer_br.get_compat_mode(); + tenant_id_ = logminer_br.get_tenant_id(); + commit_scn_ = logminer_br.get_commit_scn(); + trans_id_ = logminer_br.get_trans_id(); + orig_cluster_id_ = cdc_record->getThreadId(); + if (is_dml_record() || is_ddl_record()) { + if (OB_FAIL(fill_data_record_fields_(*cdc_record))) { + LOG_ERROR("fill data record fields failed", K(cdc_record), + K(record_type_), K(tenant_id_), K(trans_id_), K(commit_scn_)); + } + } + } + return ret; +} + +void ObLogMinerRecord::set_allocator(ObIAllocator *alloc) +{ + alloc_ = alloc; + redo_stmt_.set_allocator(alloc_); + undo_stmt_.set_allocator(alloc_); +} + +void ObLogMinerRecord::destroy() +{ + reset(); +} + +void ObLogMinerRecord::reset() +{ + is_filtered_ = false; + compat_mode_ = lib::Worker::CompatMode::INVALID; + tenant_id_ = OB_INVALID_TENANT_ID; + orig_cluster_id_ = OB_INVALID_CLUSTER_ID; + tenant_name_.reset(); + database_name_.reset(); + table_name_.reset(); + trans_id_.reset(); + record_type_ = EUNKNOWN; + commit_scn_.reset(); + + redo_stmt_.reset(); + undo_stmt_.reset(); + free_row_unique_id_(); + free_keys_(primary_keys_); + free_keys_(unique_keys_); + is_inited_ = false; +} + +bool ObLogMinerRecord::is_dml_record() const +{ + bool bret = false; + switch (record_type_) { + case EINSERT: + case EUPDATE: + case EDELETE: { + bret = true; + break; + } + default: { + bret = false; + break; + } + } + return bret; +} + +bool ObLogMinerRecord::is_ddl_record() const +{ + return EDDL == record_type_; +} + +void ObLogMinerRecord::copy_base_info(const ObLogMinerRecord &other) +{ + is_inited_ = other.is_inited_; + is_filtered_ = other.is_filtered_; + compat_mode_ = other.compat_mode_; + tenant_id_ = other.tenant_id_; + orig_cluster_id_ = other.orig_cluster_id_; + tenant_name_ = other.tenant_name_; + database_name_ = other.database_name_; + table_name_ = other.table_name_; + trans_id_ = other.trans_id_; + record_type_ = other.record_type_; + commit_scn_ = other.commit_scn_; +} + +int ObLogMinerRecord::build_stmts(ObLogMinerBR &br) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("record hasn't been initialized", KPC(this)); + } else { + ICDCRecord *cdc_rec = br.get_br(); + + if (OB_ISNULL(cdc_rec)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get a null cdc record when building redo", K(cdc_rec), KPC(this)); + } else if (is_ddl_record()) { + if (OB_FAIL(build_ddl_stmt_(*cdc_rec))) { + LOG_ERROR("build ddl stmt failed", KPC(this)); + } + } else if (is_dml_record()) { + if (OB_FAIL(build_dml_stmt_(*cdc_rec))) { + LOG_ERROR("build dml stmt failed", KPC(this)); + } + } + if (OB_SUCC(ret) && 0 == redo_stmt_.length()) { + APPEND_STMT(redo_stmt_, "/* NO SQL_REDO GENERATED */"); + } + if (OB_SUCC(ret) && 0 == undo_stmt_.length()) { + APPEND_STMT(undo_stmt_, "/* NO SQL_UNDO GENERATED */"); + } + } + return ret; +} + +void ObLogMinerRecord::free_row_unique_id_() +{ + if (nullptr != row_unique_id_.ptr()) { + alloc_->free(row_unique_id_.ptr()); + row_unique_id_.reset(); + } +} + +void ObLogMinerRecord::free_keys_(KeyArray &arr) +{ + int ret = OB_SUCCESS; + ARRAY_FOREACH(arr, idx) { + ObString &item = arr.at(idx); + alloc_->free(item.ptr()); + item.reset(); + } + arr.reset(); +} + +int ObLogMinerRecord::fill_data_record_fields_(ICDCRecord &record) +{ + int ret = OB_SUCCESS; + if (! is_dml_record() && ! is_ddl_record()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get unexpected record type when filling record fileds", K(record_type_)); + } else if (OB_FAIL(fill_tenant_db_name_(record.dbname()))) { + LOG_ERROR("fill tenant_name and db_name failed", "tenant_db_name", record.dbname()); + } else if (is_dml_record()) { + ITableMeta *tbl_meta = record.getTableMeta(); + + if (OB_ISNULL(tbl_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null table meta from cdc_record", K(tbl_meta), + K(record_type_), K(trans_id_)); + } else if (OB_FAIL(table_name_.assign(tbl_meta->getName()))) { + LOG_ERROR("failed to fill table name", "tbl_name", tbl_meta->getName()); + } else if (OB_FAIL(fill_primary_keys_(*tbl_meta))) { + LOG_ERROR("failed to fill primary keys", "tbl_name", tbl_meta->getName()); + } else if (OB_FAIL(fill_unique_keys_(*tbl_meta))) { + LOG_ERROR("failed to fill unique keys", "tbl_name", tbl_meta->getName()); + } + } + return ret; +} + +int ObLogMinerRecord::fill_row_unique_id_(ICDCRecord &cdc_record) +{ + int ret = OB_SUCCESS; + uint filter_rv_count = 0; + BinlogRecordImpl &br_impl = static_cast(cdc_record); + const binlogBuf *filter_rvs = br_impl.filterValues(filter_rv_count); + + if (nullptr != filter_rvs && filter_rv_count > 2) { + const binlogBuf &unique_id = filter_rvs[1]; + char *row_unique_id_buf = static_cast(alloc_->alloc(unique_id.buf_used_size + 1)); + if (OB_ISNULL(row_unique_id_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for row_unique_id_buf failed", K(row_unique_id_buf)); + } else { + MEMCPY(row_unique_id_buf, unique_id.buf, unique_id.buf_used_size); + row_unique_id_buf[unique_id.buf_used_size+1] = '\0'; + } + } + + return ret; +} + +int ObLogMinerRecord::fill_tenant_db_name_(const char *tenant_db_name) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(tenant_db_name)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get a null tenant_db_name", K(tenant_db_name), K(record_type_)); + } else { + // expect tenant_db_name has the format like "tenant_name.db_name" for all dml record + const char tenant_db_sep = '.'; + const int64_t tenant_db_len = STRLEN(tenant_db_name); + const char *dot_pos = STRCHR(tenant_db_name, tenant_db_sep); + if (OB_ISNULL(dot_pos)) { + if (is_dml_record()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("tenant_db_name for dml record doesn't contain dot, unexpected", K(tenant_db_name)); + } else if (is_ddl_record()) { + if (OB_FAIL(tenant_name_.assign(tenant_db_name))) { + LOG_ERROR("assign tenant name for ddl_record failed", K(tenant_db_name)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid record type when filling tenant_db_name", K(tenant_db_name), K(record_type_)); + } + } else { + const int64_t tenant_name_len = dot_pos - tenant_db_name; + ObString tenant_name(tenant_name_len, tenant_db_name); + ObString db_name(dot_pos + 1); + if (OB_FAIL(tenant_name_.assign(tenant_name))) { + LOG_ERROR("assign tenant_name failed", K(tenant_name), K(tenant_db_name)); + } else if (OB_FAIL(database_name_.assign(db_name))) { + LOG_ERROR("assign database_name failed", K(db_name), K(tenant_db_name)); + } + } + } + return ret; +} + +int ObLogMinerRecord::copy_string_to_array_(const ObString &str, + KeyArray &array) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("logminer record has not been initialized", K(is_inited_)); + } else { + const int64_t str_len = str.length(); + ObString tmp_str; + char *buf = static_cast(alloc_->alloc(str_len + 1)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate buf for obstring failed", K(buf), K(str_len), K(str)); + } else { + MEMCPY(buf, str.ptr(), str_len); + buf[str_len] = '\0'; + tmp_str.assign(buf, str_len); + if (OB_FAIL(array.push_back(tmp_str))) { + LOG_ERROR("failed to push token to keyarray", K(str), K(tmp_str), K(array)); + } + } + } + + return ret; +} + +int ObLogMinerRecord::fill_keys_(const char *key_cstr, KeyArray &key_arr) +{ + int ret = OB_SUCCESS; + const char *curr_ptr = key_cstr, *next_ptr = nullptr; + while (OB_SUCC(ret) && nullptr != (next_ptr = strchr(curr_ptr, ','))) { + ObString key(next_ptr - curr_ptr, curr_ptr); + if (OB_FAIL(copy_string_to_array_(key, key_arr))) { + LOG_ERROR("failed to copy key to primary_keys", K(key), K(key_arr)); + } else { + // skip delimiter ',' + curr_ptr = next_ptr + 1; + } + } + + if (OB_SUCC(ret)) { + // curr_ptr must end with '\0' + ObString last_key(curr_ptr); + if (OB_FAIL(copy_string_to_array_(last_key, key_arr))) { + LOG_ERROR("failed to copy last key to primary_keys_", K(last_key), K(key_arr)); + } + } + return ret; +} + +int ObLogMinerRecord::fill_primary_keys_(ITableMeta &tbl_meta) +{ + int ret = OB_SUCCESS; + const char *pks_cstr = tbl_meta.getPKs(); + if (OB_ISNULL(pks_cstr)) { + LOG_TRACE("get null pk_cstr", "table", tbl_meta.getName()); + } else { + const int64_t pks_len = STRLEN(pks_cstr); + const int64_t column_cnt = tbl_meta.getColCount(); + if (0 == pks_len) { + LOG_TRACE("no pk for tbl", K(pks_cstr), K(column_cnt), "tbl_name", tbl_meta.getName()); + } else if (OB_FAIL(fill_keys_(pks_cstr, primary_keys_))) { + LOG_ERROR("failed to fill primary keys", K(pks_cstr), "tbl_name", tbl_meta.getName()); + } else { + LOG_TRACE("finish fill primary keys", K(pks_cstr), K(primary_keys_)); + } + } + return ret; +} + +int ObLogMinerRecord::fill_unique_keys_(ITableMeta &tbl_meta) +{ + int ret = OB_SUCCESS; + const char *uks_cstr = tbl_meta.getUKs(); + if (OB_ISNULL(uks_cstr)) { + LOG_TRACE("get null uk_cstr", "table", tbl_meta.getName()); + } else { + const int64_t uks_len = STRLEN(uks_cstr); + const int64_t column_cnt = tbl_meta.getColCount(); + if (0 == uks_len) { + LOG_TRACE("no uk for tbl", K(uks_cstr), K(column_cnt), "tbl_name", tbl_meta.getName()); + } else if (OB_FAIL(fill_keys_(uks_cstr, unique_keys_))) { + LOG_ERROR("failed to fill unique keys", K(uks_cstr), "tbl_name", tbl_meta.getName()); + } else { + LOG_TRACE("finish fill unique keys", K(uks_cstr), K(unique_keys_)); + } + } + return ret; +} + +int ObLogMinerRecord::build_ddl_stmt_(ICDCRecord &cdc_rec) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("record hasn't been initialized", KPC(this)); + } else { + unsigned int new_col_cnt = 0; + binlogBuf *brbuf = cdc_rec.newCols(new_col_cnt); + if (new_col_cnt <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("new_col_cnt for ddl stmt less than zero", K(new_col_cnt), KPC(this)); + } else { + // only need ddl_stmt_str and expect it is placed in the first slot + redo_stmt_.append(brbuf[0].buf, brbuf[0].buf_used_size); + } + } + return ret; +} + +int ObLogMinerRecord::build_dml_stmt_(ICDCRecord &cdc_rec) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("record hasn't been initialized", KPC(this)); + } else { + unsigned int new_col_cnt = 0, old_col_cnt = 0; + binlogBuf *new_cols = cdc_rec.newCols(new_col_cnt); + binlogBuf *old_cols = cdc_rec.oldCols(old_col_cnt); + ITableMeta *tbl_meta = cdc_rec.getTableMeta(); + // When updating or deleting records with lob type, + // the null value of the lob type column may be incorrect + // due to the limitations of obcdc. + bool has_lob_null = false; + // xmltype and sdo_geometry type don't support compare operation. + bool has_unsupport_type_compare = false; + if (OB_SUCC(ret)) { + switch(record_type_) { + // Insert records with lob type is accurate. obcdc will output all value of lob type. + case EINSERT: { + if (OB_FAIL(build_insert_stmt_(redo_stmt_, new_cols, new_col_cnt, tbl_meta))) { + LOG_ERROR("build insert redo stmt failed", KPC(this)); + } else { + if (OB_FAIL(build_delete_stmt_(undo_stmt_, new_cols, new_col_cnt, + tbl_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build insert undo stmt failed", KPC(this)); + } else { + // ignore has_lob_null + if (has_unsupport_type_compare) { + APPEND_STMT(undo_stmt_, "/* POTENTIALLY INACCURATE */"); + } + } + } + break; + } + + // Update records with lob type maybe inaccurate, + // if NULL value appears in the pre/post mirror, the NULL may be incorrect. + case EUPDATE: { + if (OB_FAIL(build_update_stmt_(redo_stmt_, new_cols, new_col_cnt, old_cols, + old_col_cnt, tbl_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build update redo stmt failed", KPC(this)); + } else { + if (has_lob_null || has_unsupport_type_compare) { + APPEND_STMT(redo_stmt_, "/* POTENTIALLY INACCURATE */"); + has_lob_null = false; + has_unsupport_type_compare = false; + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(build_update_stmt_(undo_stmt_, old_cols, old_col_cnt, new_cols, + new_col_cnt, tbl_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build update undo stmt failed", KPC(this)); + } else { + if (has_lob_null || has_unsupport_type_compare) { + APPEND_STMT(undo_stmt_, "/* POTENTIALLY INACCURATE */"); + } + } + } + break; + } + + // Delete records with lob type maybe inaccurate, + // if NULL value appears in the pre mirror, the NULL may be incorrect. + case EDELETE: { + if (OB_FAIL(build_delete_stmt_(redo_stmt_, old_cols, old_col_cnt, + tbl_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build delete redo stmt failed", KPC(this)); + } else { + if (has_lob_null || has_unsupport_type_compare) { + APPEND_STMT(redo_stmt_, "/* POTENTIALLY INACCURATE */"); + has_lob_null = false; + has_unsupport_type_compare = false; + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(build_insert_stmt_(undo_stmt_, old_cols, + old_col_cnt, tbl_meta, has_lob_null))) { + LOG_ERROR("build delete undo stmt failed", KPC(this)); + } else { + if (has_lob_null) { + APPEND_STMT(undo_stmt_, "/* POTENTIALLY INACCURATE */"); + } + } + } + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid dml record type", KPC(this)); + break; + } + } + } + } + return ret; +} + +int ObLogMinerRecord::build_insert_stmt_(ObStringBuffer &stmt, + binlogBuf *new_cols, + const unsigned int new_col_cnt, + ITableMeta *tbl_meta) +{ + int ret = OB_SUCCESS; + // ignore has_lob_null + bool has_lob_null = false; + if (OB_FAIL(build_insert_stmt_(stmt, new_cols, new_col_cnt, tbl_meta, has_lob_null))) { + LOG_ERROR("build insert stmt failed", KPC(this)); + } + return ret; +} +int ObLogMinerRecord::build_insert_stmt_(ObStringBuffer &stmt, + binlogBuf *new_cols, + const unsigned int new_col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("record hasn't been initialized", KPC(this)); + } else { + APPEND_STMT(stmt, "INSERT INTO "); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, database_name_.ptr()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, "."); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, table_name_.ptr()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, " ("); + // append all col name + for (int i = 0; i < new_col_cnt && OB_SUCC(ret); i++) { + IColMeta *col_meta = tbl_meta->getCol(i); + if (OB_ISNULL(col_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null col_meta", "table_name", tbl_meta->getName(), K(i)); + } + if (0 != i) { + APPEND_STMT(stmt, ", "); + } + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, col_meta->getName()); + APPEND_ESCAPE_CHAR(stmt); + } + APPEND_STMT(stmt, ") VALUES ("); + // append all col value + + for (int i = 0; i < new_col_cnt && OB_SUCC(ret); i++) { + IColMeta *col_meta = tbl_meta->getCol(i); + if (OB_ISNULL(col_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null col_meta", "table_name", tbl_meta->getName(), K(i)); + } + if (0 != i) { + APPEND_STMT(stmt, ", "); + } + if (OB_SUCC(ret) && OB_FAIL(build_column_value_(stmt, col_meta, new_cols[i]))) { + LOG_ERROR("failed to build column_value", "table_name", tbl_meta->getName(), + "col_idx", i); + } + if (OB_SUCC(ret)) { + if (is_lob_type_(col_meta) && nullptr == new_cols[i].buf) { + has_lob_null = true; + } + } + } + + APPEND_STMT(stmt, ");"); + } + + if (OB_SUCC(ret)) { + LOG_TRACE("build insert stmt", "insert_stmt", stmt.ptr()); + } + return ret; +} + +int ObLogMinerRecord::build_update_stmt_(ObStringBuffer &stmt, + binlogBuf *new_cols, + const unsigned int new_col_cnt, + binlogBuf *old_cols, + const unsigned int old_col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("record hasn't been initialized", KPC(this)); + } else if (new_col_cnt != old_col_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("new_col_cnt is not equal to old_col_cnt, unexpected", K(new_col_cnt), K(old_col_cnt)); + } else { + APPEND_STMT(stmt, "UPDATE "); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, database_name_.ptr()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, "."); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, table_name_.ptr()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, " SET "); + for (int i = 0; OB_SUCC(ret) && i < new_col_cnt; i++) { + IColMeta *col_meta = tbl_meta->getCol(i); + if (OB_ISNULL(col_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null col_meta", "table_name", tbl_meta->getName(), K(i)); + } + if (0 != i) { + APPEND_STMT(stmt, ", "); + } + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, col_meta->getName()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, "="); + if (OB_SUCC(ret) && OB_FAIL(build_column_value_(stmt, col_meta, new_cols[i]))) { + LOG_ERROR("failed to build column_value", "table_name", tbl_meta->getName(), + "col_idx", i); + } + if (OB_SUCC(ret)) { + if (is_lob_type_(col_meta) && nullptr == new_cols[i].buf) { + has_lob_null = true; + } + } + } + APPEND_STMT(stmt, " WHERE "); + + if (OB_SUCC(ret) && OB_FAIL(build_where_conds_(stmt, old_cols, old_col_cnt, + tbl_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build where conds failed",); + } + if (lib::Worker::CompatMode::MYSQL == compat_mode_) { + APPEND_STMT(stmt, " LIMIT 1"); + } else if (lib::Worker::CompatMode::ORACLE == compat_mode_) { + APPEND_STMT(stmt, " AND ROWNUM=1"); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get invalid compat mode when build stmt", K(compat_mode_)); + } + + APPEND_STMT(stmt, ";"); + + if (OB_SUCC(ret)) { + LOG_TRACE("build update stmt", "update_stmt", stmt.ptr()); + } + } + return ret; +} + +int ObLogMinerRecord::build_delete_stmt_(ObStringBuffer &stmt, + binlogBuf *old_cols, + const unsigned int old_col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("record hasn't been initialized", KPC(this)); + } else { + APPEND_STMT(stmt, "DELETE FROM "); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, database_name_.ptr()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, "."); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, table_name_.ptr()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, " WHERE "); + + if (OB_SUCC(ret) && OB_FAIL(build_where_conds_(stmt, old_cols, old_col_cnt, + tbl_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build where conds failed",); + } + if (lib::Worker::CompatMode::MYSQL == compat_mode_) { + APPEND_STMT(stmt, " LIMIT 1"); + } else if (lib::Worker::CompatMode::ORACLE == compat_mode_) { + APPEND_STMT(stmt, " and ROWNUM=1"); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get invalid compat mode when build stmt", K(compat_mode_)); + } + + APPEND_STMT(stmt, ";"); + if (OB_SUCC(ret)) { + LOG_TRACE("build delete stmt", "delete_stmt", stmt.ptr()); + } + } + return ret; +} + +int ObLogMinerRecord::build_column_value_(ObStringBuffer &stmt, + IColMeta *col_meta, + binlogBuf &col_data) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(col_meta)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get null col_meta when building column_value", K(col_meta)); + } else if (OB_ISNULL(col_data.buf)) { + APPEND_STMT(stmt, "NULL"); + } else if (is_number_type_(col_meta)) { + APPEND_STMT(stmt, col_data.buf, col_data.buf_used_size); + } else if (is_binary_type_(col_meta)) { + if (lib::Worker::CompatMode::MYSQL == compat_mode_) { + APPEND_STMT(stmt, "UNHEX('"); + if (OB_FAIL(build_hex_val_(stmt, col_data))) { + LOG_ERROR("failed to build hex_val for col", "col_name", col_meta->getName()); + } + APPEND_STMT(stmt, "')"); + } else if (lib::Worker::CompatMode::ORACLE == compat_mode_) { + APPEND_STMT(stmt, "HEXTORAW('"); + if (OB_FAIL(build_hex_val_(stmt, col_data))) { + LOG_ERROR("failed to build hex_val for col", "col_name", col_meta->getName()); + } + APPEND_STMT(stmt, "')"); + } else { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("get invalid compat mode, not supported", KPC(this)); + } + } else if (is_string_type_(col_meta)) { + ObArenaAllocator tmp_alloc; + ObString target_str; + ObString src_str(col_data.buf_used_size, col_data.buf); + ObString srid; + const char *src_encoding_str = col_meta->getEncoding(); + const ObCharsetType src_cs_type = ObCharset::charset_type(src_encoding_str); + const ObCollationType src_coll_type = ObCharset::get_default_collation(src_cs_type); + const ObCollationType dst_coll_type = ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI; + + const char *curr_ptr = nullptr, *next_ptr = nullptr; + int64_t buf_size = 0; + int64_t data_len = 0; + ObStringBuffer str_val(alloc_); + bool is_atoi_valid = false; + + if (src_coll_type == dst_coll_type) { + target_str.assign_ptr(src_str.ptr(), src_str.length()); + curr_ptr = target_str.ptr(); + buf_size = target_str.length(); + } else { + if (OB_FAIL(ObCharset::charset_convert(tmp_alloc, src_str, src_coll_type, + dst_coll_type, target_str))) { + LOG_ERROR("failed to convert src_str to target_str", K(src_coll_type), K(src_encoding_str), + K(src_cs_type), K(src_coll_type), K(dst_coll_type), "src_len", src_str.length()); + } else { + curr_ptr = target_str.ptr(); + buf_size = target_str.length(); + LOG_TRACE("charset_convert finished", K(src_str), K(src_encoding_str), K(src_cs_type), + K(src_coll_type), K(dst_coll_type), K(target_str)); + } + } + if (OB_SUCC(ret)) { + if (is_geo_type_(col_meta) && lib::Worker::CompatMode::ORACLE == compat_mode_) { + if (!target_str.prefix_match(ORA_GEO_PREFIX)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("convert oracle geometry column value failed", K(target_str)); + } else { + curr_ptr += strlen(ORA_GEO_PREFIX); + buf_size -= strlen(ORA_GEO_PREFIX); + next_ptr = static_cast(memchr(curr_ptr, ';', buf_size)); + if (nullptr == next_ptr) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("convert oracle geometry column value failed", K(target_str)); + } else { + data_len = next_ptr - curr_ptr; + srid.assign_ptr(curr_ptr, data_len); + next_ptr++; + curr_ptr = next_ptr; + buf_size -= (data_len + 1); // ignore ";" + } + } + APPEND_STMT(stmt, "SDO_GEOMETRY("); + } else if (is_geo_type_(col_meta) && lib::Worker::CompatMode::MYSQL == compat_mode_) { + APPEND_STMT(stmt, "ST_GeomFromText("); + } else if (is_bit_type_(col_meta)) { + APPEND_STMT(stmt, "b"); + } + } + + APPEND_STMT(stmt, "\'"); + while (OB_SUCC(ret) && (nullptr != (next_ptr = + static_cast(memchr(curr_ptr, '\'', buf_size))))) { + // next_ptr point to the of '\'' + next_ptr++; + data_len = next_ptr - curr_ptr; + APPEND_STMT(str_val, curr_ptr, data_len); + APPEND_STMT(str_val, "'"); + curr_ptr = next_ptr; + buf_size -= data_len; + } + APPEND_STMT(str_val, curr_ptr, buf_size); + if (OB_SUCC(ret) && is_bit_type_(col_meta)) { + uint64_t bit_val = common::ObFastAtoi::atoi(str_val.ptr(), str_val.ptr() + + str_val.length(), is_atoi_valid); + if (!is_atoi_valid) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("str convert to uint64 failed failed", K(str_val)); + } else { + str_val.reset(); + if (OB_FAIL(uint_to_bit(bit_val, str_val))) { + LOG_ERROR("convert uint64 to bit failed", K(bit_val)); + } + } + } + APPEND_STMT(stmt, str_val.ptr()); + APPEND_STMT(stmt, "\'"); + if (OB_SUCC(ret)) { + if (is_geo_type_(col_meta) && lib::Worker::CompatMode::ORACLE == compat_mode_) { + APPEND_STMT(stmt, ", "); + APPEND_STMT(stmt, srid.ptr(), srid.length()); + APPEND_STMT(stmt, ")"); + } else if (is_geo_type_(col_meta) && lib::Worker::CompatMode::MYSQL == compat_mode_) { + APPEND_STMT(stmt, ")"); + } + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("get not supported column type", "col_name", col_meta->getName(), + "col_type", col_meta->getType()); + } + + return ret; +} + +int ObLogMinerRecord::build_where_conds_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare) +{ + int ret = OB_SUCCESS; + if (!unique_keys_.empty()) { + if (OB_FAIL(build_key_conds_(stmt, cols, col_cnt, tbl_meta, + unique_keys_, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build unique keys failed", K(stmt), K(unique_keys_)); + } + } else if (!primary_keys_.empty()) { + if (OB_FAIL(build_key_conds_(stmt, cols, col_cnt, tbl_meta, + primary_keys_, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build primary keys failed", K(stmt), K(primary_keys_)); + } + } else { + for (int i = 0; OB_SUCC(ret) && i < col_cnt; i++) { + IColMeta *col_meta = tbl_meta->getCol(i); + if (0 != i) { + APPEND_STMT(stmt, " AND "); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(build_cond_(stmt, cols, i, tbl_meta, col_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build cond failed", "table_name", tbl_meta->getName()); + } + } + } + } + return ret; +} + +int ObLogMinerRecord::build_key_conds_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_cnt, + ITableMeta *tbl_meta, + const KeyArray &key, + bool &has_lob_null, + bool &has_unsupport_type_compare) +{ + int ret = OB_SUCCESS; + for (int i = 0; OB_SUCC(ret) && i < key.count(); i++) { + int64_t col_idx = tbl_meta->getColIndex(key.at(i).ptr()); + if (0 > col_idx) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get col index failed", K(key)); + } else { + IColMeta *col_meta = tbl_meta->getCol(col_idx); + if (0 != i) { + APPEND_STMT(stmt, " AND "); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(build_cond_(stmt, cols, col_idx, tbl_meta, col_meta, + has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build cond failed", "table_name", tbl_meta->getName()); + } + } + } + } + return ret; +} + +int ObLogMinerRecord::build_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(col_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null col_meta", "table_name", tbl_meta->getName(), K(col_idx)); + } else { + if (is_lob_type_(col_meta) && nullptr != cols[col_idx].buf) { + // build lob type compare condition, excluding null value condition + if (OB_FAIL(build_lob_cond_(stmt, cols, col_idx, tbl_meta, + col_meta, has_lob_null, has_unsupport_type_compare))) { + LOG_ERROR("build lob condition failed", "table_name", tbl_meta->getName()); + } + } else { + if (OB_FAIL(build_normal_cond_(stmt, cols, col_idx, tbl_meta, col_meta))) { + LOG_ERROR("build normal condition failed", "table_name", tbl_meta->getName()); + } + } + if (OB_SUCC(ret)) { + if (is_lob_type_(col_meta) && nullptr == cols[col_idx].buf) { + has_lob_null = true; + } + } + } + return ret; +} + +int ObLogMinerRecord::build_lob_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_lob_type_(col_meta) || nullptr == cols[col_idx].buf)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("column type or value is unexpected", KP(col_meta), K(col_idx)); + } else { + int type = col_meta->getType(); + if (lib::Worker::CompatMode::ORACLE == compat_mode_) { + if (obmysql::MYSQL_TYPE_JSON == type) { + if (OB_FAIL(build_func_cond_(stmt, cols, col_idx, tbl_meta, col_meta, JSON_EQUAL))) { + LOG_ERROR("build func cond failed", "table_name", tbl_meta->getName(), K(JSON_EQUAL)); + } + } else if (drcmsg_field_types::DRCMSG_TYPE_ORA_XML == type || + obmysql::MYSQL_TYPE_GEOMETRY == type) { + if (OB_FAIL(build_normal_cond_(stmt, cols, col_idx, tbl_meta, col_meta))) { + LOG_ERROR("build xml condition failed", "table_name", tbl_meta->getName(), + "col_idx", col_idx); + } else { + // oracle xml and geometry type don't support compare operation. + has_unsupport_type_compare = true; + } + } else { + if (OB_FAIL(build_func_cond_(stmt, cols, col_idx, tbl_meta, col_meta, LOB_COMPARE))) { + LOG_ERROR("build func cond failed", "table_name", tbl_meta->getName(), K(LOB_COMPARE)); + } + } + } else if (lib::Worker::CompatMode::MYSQL == compat_mode_) { + if (obmysql::MYSQL_TYPE_JSON == type) { + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, col_meta->getName()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, "=cast("); + if (OB_SUCC(ret) && OB_FAIL(build_column_value_(stmt, col_meta, cols[col_idx]))) { + LOG_ERROR("failed to build column_value", "table_name", tbl_meta->getName(), + "col_idx", col_idx); + } + APPEND_STMT(stmt, "as json)"); + } else if (obmysql::MYSQL_TYPE_GEOMETRY == type) { + if (OB_FAIL(build_func_cond_(stmt, cols, col_idx, tbl_meta, col_meta, ST_EQUALS))) { + LOG_ERROR("build func cond failed", "table_name", tbl_meta->getName(), K(ST_EQUALS)); + } + } else { + if (OB_FAIL(build_normal_cond_(stmt, cols, col_idx, tbl_meta, col_meta))) { + LOG_ERROR("build xml condition failed", "table_name", tbl_meta->getName(), + "col_idx", col_idx); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("compat mode is invalid", K(compat_mode_)); + } + } + return ret; +} + +int ObLogMinerRecord::build_func_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta, + const char *func_name) +{ + int ret = OB_SUCCESS; + APPEND_STMT(stmt, func_name); + APPEND_STMT(stmt, "("); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, col_meta->getName()); + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, ", "); + if (OB_SUCC(ret) && OB_FAIL(build_column_value_(stmt, col_meta, cols[col_idx]))) { + LOG_ERROR("failed to build column_value", "table_name", tbl_meta->getName(), + "col_idx", col_idx); + } + APPEND_STMT(stmt, ")"); + return ret; +} + +int ObLogMinerRecord::build_normal_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta) +{ + int ret = OB_SUCCESS; + APPEND_ESCAPE_CHAR(stmt); + APPEND_STMT(stmt, col_meta->getName()); + APPEND_ESCAPE_CHAR(stmt); + if (OB_SUCC(ret)) { + if (nullptr == cols[col_idx].buf) { + APPEND_STMT(stmt, " IS "); + } else { + APPEND_STMT(stmt, "="); + } + } + if (OB_SUCC(ret) && OB_FAIL(build_column_value_(stmt, col_meta, cols[col_idx]))) { + LOG_ERROR("failed to build column_value", "table_name", tbl_meta->getName(), + "col_idx", col_idx); + } + return ret; +} + +int ObLogMinerRecord::build_hex_val_(ObStringBuffer &stmt, + binlogBuf &col_data) +{ + int ret = OB_SUCCESS; + char byte_buf[3] = {0}; + if (OB_FAIL(stmt.reserve(stmt.length() + col_data.buf_used_size * 2))) { + LOG_ERROR("stmt reserve failed", K(stmt), K(col_data.buf_used_size)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < col_data.buf_used_size; i++) { + // there may be a sign for signed char, 2 byte is not enough, use unsigned char instead. + unsigned char curr_byte = col_data.buf[i]; + if (OB_FAIL(databuff_printf(byte_buf, sizeof(byte_buf), "%02X", curr_byte))) { + LOG_ERROR("failed to fill byte_buf with byte", "byte", curr_byte); + } else { + APPEND_STMT(stmt, byte_buf); + } + } + + return ret; +} + +int ObLogMinerRecord::build_escape_char_(ObStringBuffer &stmt) +{ + int ret = OB_SUCCESS; + if (is_oracle_compat_mode()) { + if (OB_FAIL(stmt.append(ORACLE_ESCAPE_CHAR))) { + LOG_ERROR("append stmt failed"); + } + } else if (is_mysql_compat_mode()) { + if (OB_FAIL(stmt.append(MYSQL_ESCAPE_CHAR))) { + LOG_ERROR("append stmt failed"); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unsupported compact mode", K(compat_mode_)); + } + return ret; +} + + +bool ObLogMinerRecord::is_string_type_(IColMeta *col_meta) const +{ + int type = col_meta->getType(); + bool bret = true; + + switch(type) { + case obmysql::MYSQL_TYPE_TIMESTAMP: + case obmysql::MYSQL_TYPE_DATE: + case obmysql::MYSQL_TYPE_TIME: + case obmysql::MYSQL_TYPE_DATETIME: + case obmysql::MYSQL_TYPE_YEAR: + case obmysql::MYSQL_TYPE_NEWDATE: + case obmysql::MYSQL_TYPE_BIT: + case obmysql::MYSQL_TYPE_OB_TIMESTAMP_WITH_TIME_ZONE: + case obmysql::MYSQL_TYPE_OB_TIMESTAMP_WITH_LOCAL_TIME_ZONE: + case obmysql::MYSQL_TYPE_OB_TIMESTAMP_NANO: + case obmysql::MYSQL_TYPE_OB_NVARCHAR2: + case obmysql::MYSQL_TYPE_OB_NCHAR: + case obmysql::MYSQL_TYPE_JSON: + case obmysql::MYSQL_TYPE_ENUM: + case obmysql::MYSQL_TYPE_SET: + case obmysql::MYSQL_TYPE_ORA_CLOB: + case drcmsg_field_types::DRCMSG_TYPE_ORA_XML: + case obmysql::MYSQL_TYPE_OB_UROWID: + case obmysql::MYSQL_TYPE_OB_INTERVAL_YM: + case obmysql::MYSQL_TYPE_OB_INTERVAL_DS: + case obmysql::MYSQL_TYPE_GEOMETRY: + case obmysql::MYSQL_TYPE_OB_RAW: + bret = true; + break; + case obmysql::MYSQL_TYPE_VARCHAR: + case obmysql::MYSQL_TYPE_TINY_BLOB: + case obmysql::MYSQL_TYPE_MEDIUM_BLOB: + case obmysql::MYSQL_TYPE_LONG_BLOB: + case obmysql::MYSQL_TYPE_BLOB: + case obmysql::MYSQL_TYPE_VAR_STRING: + case obmysql::MYSQL_TYPE_STRING: + if (OB_ISNULL(col_meta->getEncoding())) { + bret = false; + LOG_WARN_RET(OB_ERR_UNEXPECTED ,"get null col encoding for string type", K(type)) + } else if (0 == strcmp(col_meta->getEncoding(), "binary")) { + bret = false; + } else { + bret = true; + } + break; + default: + bret = false; + break; + } + return bret; +} + +bool ObLogMinerRecord::is_binary_type_(IColMeta *col_meta) const +{ + obmysql::EMySQLFieldType type = static_cast(col_meta->getType()); + bool bret = false; + switch(type) { + case obmysql::MYSQL_TYPE_ORA_BLOB: + bret = true; + break; + case obmysql::MYSQL_TYPE_VARCHAR: + case obmysql::MYSQL_TYPE_TINY_BLOB: + case obmysql::MYSQL_TYPE_MEDIUM_BLOB: + case obmysql::MYSQL_TYPE_LONG_BLOB: + case obmysql::MYSQL_TYPE_BLOB: + case obmysql::MYSQL_TYPE_VAR_STRING: + case obmysql::MYSQL_TYPE_STRING: + if (OB_ISNULL(col_meta->getEncoding())) { + bret = false; + LOG_WARN_RET(OB_ERR_UNEXPECTED, "get null col encoding for string type", K(type)) + } else if (0 == strcmp(col_meta->getEncoding(), "binary")) { + bret = true; + } else { + bret = false; + } + break; + default: + bret = false; + break; + } + return bret; +} + +bool ObLogMinerRecord::is_number_type_(IColMeta *col_meta) const +{ + int type = col_meta->getType(); + bool bret = false; + switch(type) { + case obmysql::MYSQL_TYPE_DECIMAL: + case obmysql::MYSQL_TYPE_TINY: + case obmysql::MYSQL_TYPE_SHORT: + case obmysql::MYSQL_TYPE_LONG: + case obmysql::MYSQL_TYPE_FLOAT: + case obmysql::MYSQL_TYPE_DOUBLE: + case obmysql::MYSQL_TYPE_LONGLONG: + case obmysql::MYSQL_TYPE_INT24: + case obmysql::MYSQL_TYPE_COMPLEX: + case obmysql::MYSQL_TYPE_OB_NUMBER_FLOAT: + case obmysql::MYSQL_TYPE_NEWDECIMAL: + case drcmsg_field_types::DRCMSG_TYPE_ORA_BINARY_FLOAT: + case drcmsg_field_types::DRCMSG_TYPE_ORA_BINARY_DOUBLE: + bret = true; + break; + default: + bret = false; + break; + } + return bret; +} + +bool ObLogMinerRecord::is_lob_type_(IColMeta *col_meta) const +{ + int type = col_meta->getType(); + bool bret = false; + switch(type) { + case obmysql::MYSQL_TYPE_TINY_BLOB: + case obmysql::MYSQL_TYPE_MEDIUM_BLOB: + case obmysql::MYSQL_TYPE_LONG_BLOB: + case obmysql::MYSQL_TYPE_BLOB: + case obmysql::MYSQL_TYPE_ORA_BLOB: + case obmysql::MYSQL_TYPE_ORA_CLOB: + case obmysql::MYSQL_TYPE_JSON: + case obmysql::MYSQL_TYPE_GEOMETRY: + case drcmsg_field_types::DRCMSG_TYPE_ORA_XML: + bret = true; + break; + default: + bret = false; + break; + } + return bret; +} + +bool ObLogMinerRecord::is_geo_type_(IColMeta *col_meta) const +{ + obmysql::EMySQLFieldType type = static_cast(col_meta->getType()); + bool bret = false; + if (obmysql::MYSQL_TYPE_GEOMETRY == type) { + bret = true; + } + return bret; +} + +bool ObLogMinerRecord::is_bit_type_(IColMeta *col_meta) const +{ + obmysql::EMySQLFieldType type = static_cast(col_meta->getType()); + bool bret = false; + if (obmysql::MYSQL_TYPE_BIT == type) { + bret = true; + } + return bret; +} + + +} +} + +#undef APPEND_STMT +#undef APPEND_ESCAPE_CHAR \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record.h b/src/logservice/logminer/ob_log_miner_record.h new file mode 100644 index 0000000000..a7742f4ab3 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record.h @@ -0,0 +1,279 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_H_ +#define OCEANBASE_LOG_MINER_RECORD_H_ + +#include "lib/string/ob_string_buffer.h" +#include "lib/container/ob_se_array.h" +#include "lib/worker.h" +#include "rpc/obmysql/ob_mysql_global.h" +#include "storage/tx/ob_trans_define.h" + +#include "ob_log_miner_br.h" +#include "ob_log_miner_recyclable_task.h" +#include "ob_log_miner_utils.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerBR; + +typedef ObSEArray KeyArray; +class ObLogMinerRecord: public ObLogMinerRecyclableTask +{ +public: + ObLogMinerRecord(); + explicit ObLogMinerRecord(ObIAllocator *alloc); + ~ObLogMinerRecord(); + int init(ObLogMinerBR &logminer_br); + void set_allocator(ObIAllocator *alloc); + void destroy(); + void reset(); + + bool is_inited() { return is_inited_; } + + lib::Worker::CompatMode get_compat_mode() const { + return compat_mode_; + } + + bool is_mysql_compat_mode() const { + return lib::Worker::CompatMode::MYSQL == compat_mode_; + } + + bool is_oracle_compat_mode() const { + return lib::Worker::CompatMode::ORACLE == compat_mode_; + } + + uint64_t get_tenant_id() const { + return tenant_id_; + } + + int64_t get_cluster_id() const { + return orig_cluster_id_; + } + + const TenantName& get_tenant_name() const { + return tenant_name_; + } + + const DbName& get_database_name() const { + return database_name_; + } + + const TableName& get_table_name() const { + return table_name_; + } + + const transaction::ObTransID& get_ob_trans_id() const { + return trans_id_; + } + + const KeyArray& get_primary_keys() const { + return primary_keys_; + } + + const KeyArray& get_unique_keys() const { + return unique_keys_; + } + + const ObString& get_row_unique_id() const { + return row_unique_id_; + } + + RecordType get_record_type() const { + return record_type_; + } + + share::SCN get_commit_scn() const { + return commit_scn_; + } + + const ObStringBuffer& get_redo_stmt() const { + return redo_stmt_; + } + + const ObStringBuffer& get_undo_stmt() const { + return undo_stmt_; + } + + int build_stmts(ObLogMinerBR &br); + + bool is_dml_record() const; + bool is_ddl_record() const; + + void copy_base_info(const ObLogMinerRecord &other); + + TO_STRING_KV( + K(is_inited_), + K(is_filtered_), + K(compat_mode_), + K(tenant_id_), + K(tenant_name_), + K(database_name_), + K(table_name_), + K(trans_id_), + K(primary_keys_), + K(unique_keys_), + K(record_type_), + K(commit_scn_), + "redo_stmt_len", redo_stmt_.length(), + "undo_stmt_len", undo_stmt_.length() + ); + +private: + void free_keys_(KeyArray &arr); + + void free_row_unique_id_(); + + int copy_string_to_array_(const ObString &str, KeyArray &array); + + int fill_row_unique_id_(ICDCRecord &cdc_record); + + int fill_data_record_fields_(ICDCRecord &cdc_record); + + int fill_tenant_db_name_(const char *tenant_db_name); + + int fill_primary_keys_(ITableMeta &tbl_meta); + + int fill_unique_keys_(ITableMeta &tbl_meta); + + int fill_keys_(const char *key_cstr, KeyArray &key_arr); + + int build_ddl_stmt_(ICDCRecord &cdc_record); + + int build_dml_stmt_(ICDCRecord &cdc_record); + + int build_insert_stmt_(ObStringBuffer &stmt, + binlogBuf *new_cols, + const unsigned int new_col_cnt, + ITableMeta *tbl_meta); + + int build_insert_stmt_(ObStringBuffer &stmt, + binlogBuf *new_cols, + const unsigned int new_col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null); + + int build_update_stmt_(ObStringBuffer &stmt, + binlogBuf *new_cols, + const unsigned int new_col_cnt, + binlogBuf *old_cols, + const unsigned int old_col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare); + + int build_delete_stmt_(ObStringBuffer &stmt, + binlogBuf *old_cols, + const unsigned int old_col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare); + + int build_column_value_(ObStringBuffer &stmt, + IColMeta *col_meta, + binlogBuf &col_data); + + int build_where_conds_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_cnt, + ITableMeta *tbl_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare); + + int build_key_conds_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_cnt, + ITableMeta *tbl_meta, + const KeyArray &key, + bool &has_lob_null, + bool &has_unsupport_type_compare); + + int build_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare); + + int build_lob_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta, + bool &has_lob_null, + bool &has_unsupport_type_compare); + + int build_func_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta, + const char *func_name); + + int build_normal_cond_(ObStringBuffer &stmt, + binlogBuf *cols, + const unsigned int col_idx, + ITableMeta *tbl_meta, + IColMeta *col_meta); + + int build_hex_val_(ObStringBuffer &stmt, + binlogBuf &col_data); + + int build_escape_char_(ObStringBuffer &stmt); + bool is_string_type_(IColMeta *col_meta) const; + + bool is_binary_type_(IColMeta *col_meta) const; + + bool is_number_type_(IColMeta *col_meta) const; + + bool is_lob_type_(IColMeta *col_meta) const; + + bool is_geo_type_(IColMeta *col_meta) const; + + bool is_bit_type_(IColMeta *col_meta) const; + +private: + static const char *ORACLE_ESCAPE_CHAR; + static const char *MYSQL_ESCAPE_CHAR; + static const char *ORA_GEO_PREFIX; + static const char *JSON_EQUAL; + static const char *LOB_COMPARE; + static const char *ST_EQUALS; + + bool is_inited_; + bool is_filtered_; + ObIAllocator *alloc_; + + lib::Worker::CompatMode compat_mode_; + uint64_t tenant_id_; + int64_t orig_cluster_id_; + TenantName tenant_name_; + DbName database_name_; + TableName table_name_; + transaction::ObTransID trans_id_; + KeyArray primary_keys_; + KeyArray unique_keys_; + ObString row_unique_id_; + RecordType record_type_; + share::SCN commit_scn_; + ObStringBuffer redo_stmt_; + ObStringBuffer undo_stmt_; +}; +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_aggregator.cpp b/src/logservice/logminer/ob_log_miner_record_aggregator.cpp new file mode 100644 index 0000000000..b7bfbff679 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_aggregator.cpp @@ -0,0 +1,255 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_record_aggregator.h" +#include "ob_log_miner_batch_record_writer.h" +#include "ob_log_miner_batch_record.h" +#include "ob_log_miner_record.h" +#include "ob_log_miner_data_manager.h" +#include "ob_log_miner_resource_collector.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +const int64_t ObLogMinerRecordAggregator::RECORD_AGG_THREAD_NUM = 1L; +const int64_t ObLogMinerRecordAggregator::RECORD_AGG_QUEUE_SIZE = 100000L; +const int64_t ObLogMinerRecordAggregator::RECORD_FLUSH_THRESHOLD = 1L * 1000 * 1000; // 1 sec + +ObLogMinerRecordAggregator::ObLogMinerRecordAggregator(): + is_inited_(false), + cur_batch_record_(nullptr), + push_record_count_(0), + aggregated_record_count_(0), + last_trans_end_ts_(OB_INVALID_TIMESTAMP), + writer_(nullptr), + data_manager_(nullptr), + resource_collector_(nullptr), + err_handle_(nullptr) { } + +ObLogMinerRecordAggregator::~ObLogMinerRecordAggregator() +{ + destroy(); +} + +int ObLogMinerRecordAggregator::init(const int64_t start_time_us, + ILogMinerBatchRecordWriter *writer, + ILogMinerDataManager *data_manager, + ILogMinerResourceCollector *resource_collector, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("ObLogMinerRecord Aggregator has been initialized", K(is_inited_)); + } else if (OB_ISNULL(writer) || OB_ISNULL(data_manager) || OB_ISNULL(resource_collector)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("ObLogMinerRecord Aggregator get invalid argument", K(writer), K(data_manager)); + } else if (OB_FAIL(RecordAggThreadPool::init(RECORD_AGG_THREAD_NUM, RECORD_AGG_QUEUE_SIZE))) { + LOG_ERROR("RecordAggThreadPool failed to init"); + } else { + writer_ = writer; + data_manager_ = data_manager; + resource_collector_ = resource_collector; + err_handle_ = err_handle; + push_record_count_ = 0; + aggregated_record_count_ = 0; + last_trans_end_ts_ = start_time_us; + is_inited_ = true; + LOG_INFO("ObLogMinerRecordAggregator finished to init", K(writer_), K(data_manager_)); + } + + return ret; +} + +int ObLogMinerRecordAggregator::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerRecord Aggregator hasn't been initialized", K(is_inited_)); + } else if (OB_FAIL(RecordAggThreadPool::start())) { + LOG_ERROR("RecordAggThreadPool failed to start"); + } else { + LOG_INFO("ObLogMinerRecordAggregator starts"); + } + return ret; +} + +void ObLogMinerRecordAggregator::stop() +{ + RecordAggThreadPool::mark_stop_flag(); + LOG_INFO("ObLogMinerRecordAggregator stopped"); +} + +void ObLogMinerRecordAggregator::wait() +{ + RecordAggThreadPool::stop(); + LOG_INFO("ObLogMinerRecordAggregator wait end"); +} + +void ObLogMinerRecordAggregator::destroy() +{ + if (IS_INIT) { + if (nullptr != cur_batch_record_ && nullptr != data_manager_) { + data_manager_->release_logminer_batch_record(cur_batch_record_); + } + RecordAggThreadPool::destroy(); + cur_batch_record_ = nullptr; + writer_ = nullptr; + data_manager_ = nullptr; + resource_collector_ = nullptr; + err_handle_ = nullptr; + is_inited_ = false; + } +} + +int ObLogMinerRecordAggregator::push(ObLogMinerRecord *record) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("aggregator has not been initialized", K(is_inited_)); + } else if (OB_ISNULL(record)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid record in aggregator", K(record)); + } else { + LOG_TRACE("ObLogMinerRecordAggregator push record into queue", KPC(record)); + const int64_t DATA_OP_TIMEOUT = 1L * 1000 * 1000; + int64_t hash_val = 0; + RETRY_FUNC(is_stoped(), *(static_cast(this)), push, record, hash_val, DATA_OP_TIMEOUT); + if (OB_FAIL(ret)) { + LOG_ERROR("push record into MQThread queue failed", KPC(record), "is_stopped", is_stoped()); + } else { + ATOMIC_INC(&push_record_count_); + } + } + return ret; +} + +int ObLogMinerRecordAggregator::handle(void *data, const int64_t thread_index, volatile bool &stop_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("aggregator has not been initialized", K(is_inited_)); + } else if (OB_ISNULL(data)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid record in aggregator", K(data)); + } else { + ObLogMinerRecord *record = static_cast(data); + bool need_flush = false; + LOG_TRACE("aggregator process record", KPC(record)); + + if (is_trans_end_record_type(record->get_record_type())) { + last_trans_end_ts_ = record->get_commit_scn().convert_to_ts(); + } + + if (nullptr == cur_batch_record_) { + if (OB_FAIL(data_manager_->get_logminer_batch_record(cur_batch_record_))) { + LOG_ERROR("get batch record from data manager failed", KPC(cur_batch_record_)); + } else if (OB_ISNULL(cur_batch_record_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("get null batch record from data_manager", KPC(cur_batch_record_)); + } else if (OB_FAIL(cur_batch_record_->init_last_trans_end_ts(last_trans_end_ts_))) { + LOG_ERROR("failed to init last trans end ts", KPC(cur_batch_record_)); + } + } + + if (OB_FAIL(ret)) { + LOG_ERROR("failed to get cur_batch_record"); + } else if (OB_FAIL(cur_batch_record_->append_record(*record))) { + LOG_ERROR("append record to cur_batch_record failed", KPC(record), KPC(cur_batch_record_)); + } else if (OB_FAIL(check_need_flush_(need_flush))) { + LOG_ERROR("check need flush failed", KPC(cur_batch_record_)); + } else if (OB_FAIL(resource_collector_->revert(record))) { + LOG_ERROR("resource collector failed to revert record", KP(record)); + } else { + LOG_TRACE("batch_record need flush", K(need_flush)); + if (need_flush) { + ATOMIC_AAF(&aggregated_record_count_, cur_batch_record_->get_total_record_count()); + if (OB_FAIL(writer_->push(cur_batch_record_))) { + LOG_ERROR("push cur_batch_record into writer failed", KPC(cur_batch_record_)); + } else { + cur_batch_record_ = nullptr; + } + } + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "ObLogMinerRecordAggregator exit unexpected\n"); + } + + return ret; +} + +int ObLogMinerRecordAggregator::get_total_record_count(int64_t &record_count) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("aggregator has not been initialized", K(is_inited_)); + } else { + record_count = ATOMIC_LOAD(&push_record_count_) - ATOMIC_LOAD(&aggregated_record_count_); + } + + return ret; +} + +int ObLogMinerRecordAggregator::check_need_flush_(bool &need_flush) +{ + int ret = OB_SUCCESS; + const char *reason = "INVALID"; + int64_t first_access_ts = OB_INVALID_TIMESTAMP; + need_flush = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("aggregator has not been initialized", K(is_inited_)); + } else if (OB_ISNULL(cur_batch_record_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cur_batch_record_ is null when check_need_flush", K(cur_batch_record_)); + } else if (cur_batch_record_->is_freezed()) { + need_flush = true; + reason = "Freezed"; + } else if (! cur_batch_record_->get_progress_range().is_valid()) { + need_flush = false; + reason = "Invalid Progress Range"; + } else if (data_manager_->reach_end_progress(cur_batch_record_->get_last_trans_end_ts())) { + need_flush = true; + reason = "Reach End Progress"; + LOG_INFO("cur_bacth_record reached end progress", KPC(cur_batch_record_), K(need_flush)); + } else { + first_access_ts = cur_batch_record_->get_first_access_ts(); + if (OB_INVALID_TIMESTAMP == first_access_ts) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cur_batch_record hasn't been accessed when check_need_flush", KPC(cur_batch_record_)); + } else if (ObTimeUtility::current_time() - first_access_ts >= RECORD_FLUSH_THRESHOLD) { + need_flush = true; + reason = "Time Threshold"; + } else { + reason = "Wait Flush"; + } + } + + LOG_TRACE("check_need_flush end", KR(ret), K(need_flush), K(reason), K(first_access_ts)); + + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_aggregator.h b/src/logservice/logminer/ob_log_miner_record_aggregator.h new file mode 100644 index 0000000000..73f10034db --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_aggregator.h @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_AGGREGATOR_H_ +#define OCEANBASE_LOG_MINER_RECORD_AGGREGATOR_H_ + +#include "lib/thread/ob_multi_fixed_queue_thread.h" +#include "ob_log_miner_error_handler.h" +#include "ob_log_miner_batch_record.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerRecord; +class ILogMinerAnalysisWriter; +class ILogMinerDataManager; +class ILogMinerBatchRecordWriter; +class ILogMinerResourceCollector; + +class ILogMinerRecordAggregator +{ +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int push(ObLogMinerRecord *record) = 0; + virtual int get_total_record_count(int64_t &record_count) = 0; +}; + +typedef common::ObMQThread<1, ILogMinerRecordAggregator> RecordAggThreadPool; + +class ObLogMinerRecordAggregator: public ILogMinerRecordAggregator, public RecordAggThreadPool +{ +public: + static const int64_t RECORD_AGG_THREAD_NUM; + static const int64_t RECORD_AGG_QUEUE_SIZE; + static const int64_t RECORD_FLUSH_THRESHOLD; +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int push(ObLogMinerRecord *record); + virtual int get_total_record_count(int64_t &record_count); +public: + virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag); + +public: + ObLogMinerRecordAggregator(); + ~ObLogMinerRecordAggregator(); + + int init(const int64_t start_time_us, + ILogMinerBatchRecordWriter *writer, + ILogMinerDataManager *data_manager, + ILogMinerResourceCollector *resource_collector, + ILogMinerErrorHandler *err_handle); + +private: + int check_need_flush_(bool &need_flush); + +private: + bool is_inited_; + ObLogMinerBatchRecord *cur_batch_record_; + int64_t push_record_count_ CACHE_ALIGNED; + int64_t aggregated_record_count_ CACHE_ALIGNED; + int64_t last_trans_end_ts_; + ILogMinerBatchRecordWriter *writer_; + ILogMinerDataManager *data_manager_; + ILogMinerResourceCollector *resource_collector_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_converter.cpp b/src/logservice/logminer/ob_log_miner_record_converter.cpp new file mode 100644 index 0000000000..66fa80bf5a --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_converter.cpp @@ -0,0 +1,445 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "lib/json_type/ob_json_base.h" // ObJsonBaseUtil +#include "lib/timezone/ob_time_convert.h" // ObTimeConverter +#include "ob_log_miner_record_converter.h" +#include "logservice/common_util/ob_log_time_utils.h" +#include "ob_log_miner_logger.h" +#include "ob_log_miner_utils.h" + +#define APPEND_STR(buf, args...) \ + do {\ + if (OB_SUCC(ret) && OB_FAIL(buf.append(args))) { \ + LOG_ERROR("append str failed", "buf_len", buf.length(), "buf_cap", buf.capacity()); \ + }\ + } while(0) + +namespace oceanbase +{ +namespace oblogminer +{ +#define MINER_SCHEMA_DEF(field, id, args...) \ +ILogMinerRecordConverter::ColType::field, +const ILogMinerRecordConverter::ColType ILogMinerRecordConverter::COL_ORDER[] = +{ + #include "ob_log_miner_analyze_schema.h" +}; +#undef MINER_SCHEMA_DEF + +const char *ILogMinerRecordConverter::DELIMITER = ","; + +ILogMinerRecordConverter *ILogMinerRecordConverter::get_converter_instance(const RecordFileFormat format) +{ + static ObLogMinerRecordCsvConverter csv_converter; + static ObLogMinerRecordRedoSqlConverter redo_sql_converter; + static ObLogMinerRecordUndoSqlConverter undo_sql_converter; + static ObLogMinerRecordJsonConverter json_converter; + + ILogMinerRecordConverter *converter = nullptr; + switch(format) { + case RecordFileFormat::CSV: { + converter = &csv_converter; + break; + } + + case RecordFileFormat::REDO_ONLY: { + converter = &redo_sql_converter; + break; + } + + case RecordFileFormat::UNDO_ONLY: { + converter = &undo_sql_converter; + break; + } + + case RecordFileFormat::JSON: { + converter = &json_converter; + break; + } + + default: { + converter = nullptr; + break; + } + } + + return converter; +} + +////////////////////////////// ObLogMinerRecordCsvConverter ////////////////////////////// + +int ObLogMinerRecordCsvConverter::write_record(const ObLogMinerRecord &record, + common::ObStringBuffer &buffer, bool &is_written) +{ + int ret = OB_SUCCESS; + + const int64_t col_num = sizeof(COL_ORDER) / sizeof(ColType); + + for (int64_t i = 0; i < col_num && OB_SUCC(ret); i++) { + const char *end_char = i == col_num-1 ? "\n": DELIMITER; + switch(COL_ORDER[i]) { + case ColType::TENANT_ID: { + if (OB_FAIL(write_unsigned_number(record.get_tenant_id(), buffer))) { + LOG_ERROR("write tenant_id failed", K(record)); + } // tenant_id + break; + } + + case ColType::TRANS_ID: { + if (OB_FAIL(write_signed_number(record.get_ob_trans_id().get_id(), buffer))) { + LOG_ERROR("write trans_id failed", K(record)); + } // trans_id + break; + } + + case ColType::PRIMARY_KEY: { + if (OB_FAIL(write_keys(record.get_primary_keys(), buffer))) { + LOG_ERROR("write primary_key failed", K(record)); + } // primary_key + break; + } + + case ColType::TENANT_NAME: { + if (OB_FAIL(write_string_no_escape(record.get_tenant_name().str(), buffer))) { + LOG_ERROR("write tenant_name failed", K(record)); + } // tenant_name + break; + } + + case ColType::DATABASE_NAME: { + if (OB_FAIL(write_string_no_escape(record.get_database_name().str(), buffer))) { + LOG_ERROR("write database_name failed", K(record)); + } // database_name + break; + } + + case ColType::TABLE_NAME: { + if (OB_FAIL(write_string_no_escape(record.get_table_name().str(), buffer))) { + LOG_ERROR("write table_name failed", K(record)); + } // table_name + break; + } + + case ColType::OPERATION: { + if (OB_FAIL(write_string_no_escape(record_type_to_str(record.get_record_type()), buffer))) { + LOG_ERROR("write operation failed", K(record)); + } // operation + break; + } + + case ColType::OPERATION_CODE: { + if (OB_FAIL(write_signed_number(record_type_to_num(record.get_record_type()), buffer))) { + LOG_ERROR("write operation_code failed", K(record)); + } // operation_code + break; + } + + case ColType::COMMIT_SCN: { + if (OB_FAIL(write_signed_number(record.get_commit_scn().get_val_for_inner_table_field(), buffer))) { + LOG_ERROR("write commit_scn failed", K(record)); + } // commit scn + break; + } + + case ColType::COMMIT_TIMESTAMP: { + const int16_t scale = 6; + char time_buf[128] = {0}; + int64_t pos = 0; + ObString nls_format; + if (OB_FAIL(ObTimeConverter::datetime_to_str(record.get_commit_scn().convert_to_ts(), + &LOGMINER_TZ.get_tz_info(), nls_format, scale, time_buf, sizeof(time_buf), pos))) { + LOG_ERROR("failed to get time string from commit_scn", K(record)); + } else if (OB_FAIL(write_string_no_escape(time_buf, buffer))) { + LOG_ERROR("write commit_timestamp failed", K(record)); + } + break; + } + + case ColType::SQL_REDO: { + if (OB_FAIL(write_csv_string_escape_(record.get_redo_stmt().string(), buffer))) { + LOG_ERROR("write redo_stmt failed", K(record)); + } // redo_stmt + break; + } + + case ColType::SQL_UNDO: { + if (OB_FAIL(write_csv_string_escape_(record.get_undo_stmt().string(), buffer))) { + LOG_ERROR("write undo_stmt failed", K(record)); + } // undo_stmt + break; + } + + case ColType::ORG_CLUSTER_ID: { + if (OB_FAIL(write_signed_number(record.get_cluster_id(), buffer))) { + LOG_ERROR("write org_cluster_id failed", K(record)); + } // org_cluster_id + break; + } + + default: { + ret = OB_ERR_DEFENSIVE_CHECK; + LOG_ERROR("unsupported column type", K(COL_ORDER[i]), K(record)); + break; + } + } + + if (OB_SUCC(ret) && OB_FAIL(buffer.append(end_char))) { + LOG_ERROR("append delimiter failed", K(record), K(end_char)); + } + } + + if (OB_SUCC(ret)) { + is_written = true; + LOG_TRACE("ObLogMinerRecordCsvConverter write record succ", K(record)); + } + + return ret; +} + +int ObLogMinerRecordCsvConverter::write_csv_string_escape_(const ObString &str, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + const char *data = str.ptr(), *prev_ptr = data; + APPEND_STR(buffer, "\""); + while (OB_SUCC(ret) && nullptr != prev_ptr && nullptr != (data = strchr(prev_ptr, '"'))) { + APPEND_STR(buffer, prev_ptr, data - prev_ptr + 1); + APPEND_STR(buffer, "\""); + prev_ptr = data + 1; + } + + APPEND_STR(buffer, prev_ptr); + APPEND_STR(buffer, "\""); + return ret; +} + +////////////////////////////// ObLogMinerRecordRedoSqlConverter ////////////////////////////// + +int ObLogMinerRecordRedoSqlConverter::write_record(const ObLogMinerRecord &record, + common::ObStringBuffer &buffer, bool &is_written) +{ + int ret = OB_SUCCESS; + if (!record.get_redo_stmt().empty()) { + APPEND_STR(buffer, record.get_redo_stmt().string()); + APPEND_STR(buffer, "\n"); + if (OB_SUCC(ret)) { + is_written = true; + LOG_TRACE("ObLogMinerRecordRedoSqlConverter write record succ", K(record)); + } + } + return ret; +} + +////////////////////////////// ObLogMinerRecordUndoSqlConverter ////////////////////////////// + +int ObLogMinerRecordUndoSqlConverter::write_record(const ObLogMinerRecord &record, + common::ObStringBuffer &buffer, bool &is_written) +{ + int ret = OB_SUCCESS; + if (!record.get_undo_stmt().empty()) { + APPEND_STR(buffer, record.get_undo_stmt().string()); + APPEND_STR(buffer, "\n"); + if (OB_SUCC(ret)) { + is_written = true; + LOG_TRACE("ObLogMinerRecordUndoSqlConverter write record succ", K(record)); + } + } + return ret; +} + +////////////////////////////// ObLogMinerRecordJsonConverter ////////////////////////////// + +int ObLogMinerRecordJsonConverter::write_record(const ObLogMinerRecord &record, + common::ObStringBuffer &buffer, bool &is_written) +{ + int ret = OB_SUCCESS; + + const int64_t col_num = sizeof(COL_ORDER) / sizeof(ColType); + APPEND_STR(buffer, "{"); + + for (int64_t i = 0; i < col_num && OB_SUCC(ret); i++) { + const char *end_char = i == col_num-1 ? "}\n": DELIMITER; + switch(COL_ORDER[i]) { + case ColType::TENANT_ID: { + if (OB_FAIL(write_json_key_("TENANT_ID", buffer))) { + LOG_ERROR("write json_key TENANT_ID failed", K(record)); + } else if (OB_FAIL(write_unsigned_number(record.get_tenant_id(), buffer))) { + LOG_ERROR("write tenant_id failed", K(record)); + } // tenant_id + break; + } + + case ColType::TRANS_ID: { + if (OB_FAIL(write_json_key_("TRANS_ID", buffer))) { + LOG_ERROR("write json_key TRANS_ID failed", K(record)); + } else if (OB_FAIL(write_signed_number(record.get_ob_trans_id().get_id(), buffer))) { + LOG_ERROR("write trans_id failed", K(record)); + } // trans_id + break; + } + + case ColType::PRIMARY_KEY: { + if (OB_FAIL(write_json_key_("PRIMARY_KEY", buffer))) { + LOG_ERROR("write json_key PRIMARY_KEY failed", K(record)); + } else if (OB_FAIL(write_keys(record.get_primary_keys(), buffer))) { + LOG_ERROR("write primary_key failed", K(record)); + } // primary_key + break; + } + + case ColType::TENANT_NAME: { + if (OB_FAIL(write_json_key_("TENANT_NAME", buffer))) { + LOG_ERROR("write json_key TENANT_NAME failed", K(record)); + } else if (OB_FAIL(write_string_no_escape(record.get_tenant_name().str(), buffer))) { + LOG_ERROR("write tenant_name failed", K(record)); + } // tenant_name + break; + } + + case ColType::DATABASE_NAME: { + if (OB_FAIL(write_json_key_("DATABASE_NAME", buffer))) { + LOG_ERROR("write json_key DATABASE_NAME failed", K(record)); + } else if (OB_FAIL(write_string_no_escape(record.get_database_name().str(), buffer))) { + LOG_ERROR("write database_name failed", K(record)); + } // database_name + break; + } + + case ColType::TABLE_NAME: { + if (OB_FAIL(write_json_key_("TABLE_NAME", buffer))) { + LOG_ERROR("write json_key TABLE_NAME failed", K(record)); + } else if (OB_FAIL(write_string_no_escape(record.get_table_name().str(), buffer))) { + LOG_ERROR("write table_name failed", K(record)); + } // table_name + break; + } + + case ColType::OPERATION: { + if (OB_FAIL(write_json_key_("OPERATION", buffer))) { + LOG_ERROR("write json_key OPERATION failed", K(record)); + } else if (OB_FAIL(write_string_no_escape(record_type_to_str(record.get_record_type()), buffer))) { + LOG_ERROR("write operation failed", K(record)); + } // operation + break; + } + + case ColType::OPERATION_CODE: { + if (OB_FAIL(write_json_key_("OPERATION_CODE", buffer))) { + LOG_ERROR("write json_key OPERATION_CODE failed", K(record)); + } else if (OB_FAIL(write_signed_number(record_type_to_num(record.get_record_type()), buffer))) { + LOG_ERROR("write operation_code failed", K(record)); + } // operation_code + break; + } + + case ColType::COMMIT_SCN: { + if (OB_FAIL(write_json_key_("COMMIT_SCN", buffer))) { + LOG_ERROR("write json_key COMMIT_SCN failed", K(record)); + } else if (OB_FAIL(write_signed_number(record.get_commit_scn().get_val_for_inner_table_field(), buffer))) { + LOG_ERROR("write commit_scn failed", K(record)); + } // commit scn + break; + } + + case ColType::COMMIT_TIMESTAMP: { + const int16_t scale = 6; + char time_buf[128] = {0}; + int64_t pos = 0; + ObString nls_format; + if (OB_FAIL(write_json_key_("COMMIT_TIMESTAMP", buffer))) { + LOG_ERROR("write json_key COMMIT_TIMESTAMP failed", K(record)); + } else if (OB_FAIL(ObTimeConverter::datetime_to_str(record.get_commit_scn().convert_to_ts(), + &LOGMINER_TZ.get_tz_info(), nls_format, scale, time_buf, sizeof(time_buf), pos))) { + LOG_ERROR("failed to get time string from commit_scn", K(record)); + } else if (OB_FAIL(write_string_no_escape(time_buf, buffer))) { + LOG_ERROR("write commit_timestamp failed", K(record)); + } + break; + } + + case ColType::SQL_REDO: { + if (OB_FAIL(write_json_key_("SQL_REDO", buffer))) { + LOG_ERROR("write json_key SQL_REDO failed", K(record)); + } else if (OB_FAIL(write_json_string_escape_(record.get_redo_stmt().string(), buffer))) { + LOG_ERROR("write redo_stmt failed", K(record)); + } // redo_stmt + break; + } + + case ColType::SQL_UNDO: { + if (OB_FAIL(write_json_key_("SQL_UNDO", buffer))) { + LOG_ERROR("write json_key SQL_UNDO failed", K(record)); + } else if (OB_FAIL(write_json_string_escape_(record.get_undo_stmt().string(), buffer))) { + LOG_ERROR("write undo_stmt failed", K(record)); + } // undo_stmt + break; + } + + case ColType::ORG_CLUSTER_ID: { + if (OB_FAIL(write_json_key_("ORG_CLUSTER_ID", buffer))) { + LOG_ERROR("write json_key ORG_CLUSTER_ID failed", K(record)); + } else if (OB_FAIL(write_signed_number(record.get_cluster_id(), buffer))) { + LOG_ERROR("write org_cluster_id failed", K(record)); + } // org_cluster_id + break; + } + + default: { + ret = OB_ERR_DEFENSIVE_CHECK; + LOG_ERROR("unsupported column type", K(COL_ORDER[i]), K(record)); + break; + } + } + + if (OB_SUCC(ret) && OB_FAIL(buffer.append(end_char))) { + LOG_ERROR("append delimiter failed", K(record), K(end_char)); + } + + if (OB_SUCC(ret)) { + is_written = true; + LOG_TRACE("ObLogMinerRecordJsonConverter write record succ", K(record)); + } + } + + return ret; +} + +int ObLogMinerRecordJsonConverter::write_json_key_(const char *str, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + APPEND_STR(buffer, "\""); + APPEND_STR(buffer, str); + APPEND_STR(buffer, "\":"); + return ret; +} + +int ObLogMinerRecordJsonConverter::write_json_string_escape_(const ObString &str, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + const char *data = str.ptr(); + if (nullptr != data) { + if (OB_FAIL(ObJsonBaseUtil::add_double_quote(buffer, data, str.length()))) { + LOG_ERROR("add_double_quote failed", K(str)); + } + } else { + APPEND_STR(buffer, "\"\""); + } + + return ret; +} + +} +} + +#undef APPEND_STR \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_converter.h b/src/logservice/logminer/ob_log_miner_record_converter.h new file mode 100644 index 0000000000..52fdcb7676 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_converter.h @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_CONVERTER_H_ +#define OCEANBASE_LOG_MINER_RECORD_CONVERTER_H_ + +#include "ob_log_miner_file_manager.h" +#include "ob_log_miner_record.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerRecordConverter +{ +public: + // TODO: there may be concurrency issue in future? + static ILogMinerRecordConverter *get_converter_instance(const RecordFileFormat format); + virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written) = 0; + +public: + // TENANT_ID,TRANS_ID,PRIMARY_KEY,ROW_UNIQUE_ID,SEQ_NO,TENANT_NAME,USER_NAME,TABLE_NAME,OPERATION, + // OPERATION_CODE,COMMIT_SCN,COMMIT_TIMESTAMP,SQL_REDO,SQL_UNDO,ORG_CLUSTER_ID + #define MINER_SCHEMA_DEF(field, id, args...) \ + field = id, + enum class ColType { + #include "ob_log_miner_analyze_schema.h" + }; + #undef MINER_SCHEMA_DEF + + const static ColType COL_ORDER[]; + static const char *DELIMITER; +}; + +class ObLogMinerRecordCsvConverter: public ILogMinerRecordConverter +{ +public: + virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written); +public: + ObLogMinerRecordCsvConverter() {}; + ~ObLogMinerRecordCsvConverter() {} +private: + int write_csv_string_escape_(const ObString &str, common::ObStringBuffer &buffer); +}; + +class ObLogMinerRecordRedoSqlConverter: public ILogMinerRecordConverter +{ +public: + virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written); +public: + ObLogMinerRecordRedoSqlConverter() {} + ~ObLogMinerRecordRedoSqlConverter() {} + +}; + +class ObLogMinerRecordUndoSqlConverter: public ILogMinerRecordConverter +{ +public: + virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written); +public: + ObLogMinerRecordUndoSqlConverter() {} + ~ObLogMinerRecordUndoSqlConverter() {} +}; + +class ObLogMinerRecordJsonConverter: public ILogMinerRecordConverter +{ +public: + virtual int write_record(const ObLogMinerRecord &record, common::ObStringBuffer &buffer, bool &is_written); +public: + ObLogMinerRecordJsonConverter() {}; + ~ObLogMinerRecordJsonConverter() {} +private: + int write_json_key_(const char *str, common::ObStringBuffer &buffer); + int write_json_string_escape_(const ObString &str, common::ObStringBuffer &buffer); +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_file_format.cpp b/src/logservice/logminer/ob_log_miner_record_file_format.cpp new file mode 100644 index 0000000000..6eafb2afaf --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_file_format.cpp @@ -0,0 +1,73 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_miner_record_file_format.h" +#include +namespace oceanbase +{ +namespace oblogminer +{ +RecordFileFormat get_record_file_format(const common::ObString &file_format_str) +{ + RecordFileFormat format = RecordFileFormat::INVALID; + if (0 == file_format_str.case_compare("CSV")) { + format = RecordFileFormat::CSV; + } else if (0 == file_format_str.case_compare("REDO_ONLY")) { + format = RecordFileFormat::REDO_ONLY; + } else if (0 == file_format_str.case_compare("UNDO_ONLY")) { + format = RecordFileFormat::UNDO_ONLY; + } else if (0 == file_format_str.case_compare("JSON")) { + format = RecordFileFormat::JSON; + } else if (0 == file_format_str.case_compare("AVRO")) { + format = RecordFileFormat::AVRO; + } else if (0 == file_format_str.case_compare("PARQUET")) { + format = RecordFileFormat::PARQUET; + } + return format; +} + +RecordFileFormat get_record_file_format(const char *file_format_str) +{ + return get_record_file_format(common::ObString(file_format_str)); +} +const char *record_file_format_str(const RecordFileFormat format) { + const char *result = nullptr; + switch (format) { + case RecordFileFormat::INVALID: + result = "INVALID"; + break; + case RecordFileFormat::CSV: + result = "CSV"; + break; + case RecordFileFormat::REDO_ONLY: + result = "REDO_ONLY"; + break; + case RecordFileFormat::UNDO_ONLY: + result = "UNDO_ONLY"; + break; + case RecordFileFormat::JSON: + result = "JSON"; + break; + case RecordFileFormat::PARQUET: + result = "PARQUET"; + break; + case RecordFileFormat::AVRO: + result = "AVRO"; + break; + default: + result = "NOT_SUPPORTED"; + break; + } + return result; +} +} +} diff --git a/src/logservice/logminer/ob_log_miner_record_file_format.h b/src/logservice/logminer/ob_log_miner_record_file_format.h new file mode 100644 index 0000000000..e3cb5117f8 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_file_format.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_FILE_FORMAT_H_ +#define OCEANBASE_LOG_MINER_RECORD_FILE_FORMAT_H_ + +#include "lib/string/ob_string.h" +namespace oceanbase +{ +namespace oblogminer +{ +enum class RecordFileFormat +{ + INVALID = -1, + CSV = 0, + REDO_ONLY, + UNDO_ONLY, + JSON, + PARQUET, + AVRO +}; +RecordFileFormat get_record_file_format(const common::ObString &file_format_str); + +RecordFileFormat get_record_file_format(const char *file_format_str); + +const char *record_file_format_str(const RecordFileFormat format); +} +} +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_filter.cpp b/src/logservice/logminer/ob_log_miner_record_filter.cpp new file mode 100644 index 0000000000..e981fc55c6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_filter.cpp @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_filter.h b/src/logservice/logminer/ob_log_miner_record_filter.h new file mode 100644 index 0000000000..5556c61ac4 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_filter.h @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_FILTER_H_ +#define OCEANBASE_LOG_MINER_RECORD_FILTER_H_ + +#include "lib/string/ob_string.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerRecord; + +class ILogMinerRecordFilter { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void destroy() = 0; + virtual int push(const ObLogMinerRecord &record) = 0; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_parser.cpp b/src/logservice/logminer/ob_log_miner_record_parser.cpp new file mode 100644 index 0000000000..e981fc55c6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_parser.cpp @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_parser.h b/src/logservice/logminer/ob_log_miner_record_parser.h new file mode 100644 index 0000000000..16fb5b4399 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_parser.h @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_PARSER_H_ +#define OCEANBASE_LOG_MINER_RECORD_PARSER_H_ + +#include "lib/string/ob_string.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ILogMinerRecordParser { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void destroy() = 0; + virtual int push(const common::ObString &record_str) = 0; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_rewriter.cpp b/src/logservice/logminer/ob_log_miner_record_rewriter.cpp new file mode 100644 index 0000000000..e981fc55c6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_rewriter.cpp @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_record_rewriter.h b/src/logservice/logminer/ob_log_miner_record_rewriter.h new file mode 100644 index 0000000000..dba9d69219 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_record_rewriter.h @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECORD_REWRITER_H_ +#define OCEANBASE_LOG_MINER_RECORD_REWRITER_H_ + +#include "lib/string/ob_string.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerUndoTask; + +class ILogMinerRecordRewriter { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void destroy() = 0; + virtual int push(ObLogMinerUndoTask *task) = 0; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_recyclable_task.h b/src/logservice/logminer/ob_log_miner_recyclable_task.h new file mode 100644 index 0000000000..ac63a7b168 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_recyclable_task.h @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RECYCLABLE_TASK_H_ +#define OCEANBASE_LOG_MINER_RECYCLABLE_TASK_H_ + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerRecyclableTask +{ +public: + enum class TaskType + { + UNKNOWN = 0, + BINLOG_RECORD, + LOGMINER_RECORD, + BATCH_RECORD, + UNDO_TASK + }; + + explicit ObLogMinerRecyclableTask(TaskType type): type_(type) { } + ~ObLogMinerRecyclableTask() { type_ = TaskType::UNKNOWN; } + + bool is_binlog_record() const { + return TaskType::BINLOG_RECORD == type_; + } + bool is_logminer_record() const { + return TaskType::LOGMINER_RECORD == type_; + } + bool is_batch_record() const { + return TaskType::BATCH_RECORD == type_; + } + bool is_undo_task() const { + return TaskType::UNDO_TASK == type_; + } + + TaskType get_task_type() const + { + return type_; + } + +protected: + TaskType type_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_resource_collector.cpp b/src/logservice/logminer/ob_log_miner_resource_collector.cpp new file mode 100644 index 0000000000..f7301d568c --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_resource_collector.cpp @@ -0,0 +1,218 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_resource_collector.h" +#include "ob_log_miner_recyclable_task.h" +#include "ob_log_miner_br.h" +#include "ob_log_miner_record.h" +#include "ob_log_miner_batch_record.h" +#include "ob_log_miner_data_manager.h" +#include "ob_log_miner_logger.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +const int64_t ObLogMinerResourceCollector::RC_THREAD_NUM = 1L; +const int64_t ObLogMinerResourceCollector::RC_QUEUE_SIZE = 100000L; + +ObLogMinerResourceCollector::ObLogMinerResourceCollector(): + is_inited_(false), + data_manager_(nullptr), + err_handle_(nullptr) +{ +} + +ObLogMinerResourceCollector::~ObLogMinerResourceCollector() +{ + destroy(); +} + +int ObLogMinerResourceCollector::init(ILogMinerDataManager *data_manager, + ILogMinerErrorHandler *err_handle) +{ + int ret = OB_SUCCESS; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_ERROR("Resource Collector has been initialized", K(is_inited_)); + } else if (OB_ISNULL(data_manager)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get invalid argument when trying to initialize resource collector", K(data_manager)); + } else if (OB_FAIL(ResourceCollectorThread::init(RC_THREAD_NUM, RC_QUEUE_SIZE))) { + LOG_ERROR("failed to init resource collector thread", K(RC_THREAD_NUM), K(RC_QUEUE_SIZE)); + } else { + is_inited_ = true; + data_manager_ = data_manager; + err_handle_ = err_handle_; + LOG_INFO("ObLogMinerResourceCollector finished to init"); + LOGMINER_STDOUT_V("ObLogMinerResourceCollector finished to init\n"); + } + + return ret; +} + +int ObLogMinerResourceCollector::start() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_)); + } else if (OB_FAIL(ResourceCollectorThread::start())) { + LOG_ERROR("ResourceCollectorThread failed to start"); + } else { + LOG_INFO("ObLogMinerResourceCollector starts"); + } + + return ret; +} + +void ObLogMinerResourceCollector::stop() +{ + ResourceCollectorThread::mark_stop_flag(); + LOG_INFO("ObLogMinerResourceCollector stopped"); +} + +void ObLogMinerResourceCollector::wait() +{ + ResourceCollectorThread::stop(); + LOG_INFO("ObLogMinerResourceCollector finished to wait"); +} + +void ObLogMinerResourceCollector::destroy() +{ + if (IS_INIT) { + is_inited_ = false; + data_manager_ = nullptr; + err_handle_ = nullptr; + LOG_INFO("ObLogMinerResourceCollector destroyed"); + LOGMINER_STDOUT_V("ObLogMinerResourceCollector destroyed\n"); + } +} + +int ObLogMinerResourceCollector::revert(ObLogMinerRecyclableTask *task) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_)); + } else if (OB_FAIL(push_task_(task))) { + LOG_ERROR("failed to push task into thread queue", K(task)); + } + return ret; +} + +int ObLogMinerResourceCollector::get_total_task_count(int64_t &task_count) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_)); + } else if (OB_FAIL(ResourceCollectorThread::get_total_task_num(task_count))) { + LOG_ERROR("failed to get total task count in resource collector"); + } + + return ret; +} + +int ObLogMinerResourceCollector::handle(void *data, const int64_t thread_index, volatile bool &stop_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObLogMinerResourceCollector hasn't been initialized", K(is_inited_)); + } else if (OB_ISNULL(data)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("get null data when handling data", K(data)); + } else { + ObLogMinerRecyclableTask *task = static_cast(data); + if (task->is_logminer_record()) { + ObLogMinerRecord *rec = static_cast(task); + if (OB_FAIL(handle_logminer_record_(rec))) { + LOG_ERROR("failed to handle logminer record", KPC(rec)); + } + } else if (task->is_binlog_record()) { + ObLogMinerBR *br = static_cast(task); + if (OB_FAIL(handle_binlog_record_(br))) { + LOG_ERROR("failed to handle binlog record", KPC(br)); + } + } else if (task->is_batch_record()) { + ObLogMinerBatchRecord *batch_rec = static_cast(task); + if (OB_FAIL(handle_batch_record_(batch_rec))) { + LOG_ERROR("failed to handle batch record", KPC(batch_rec)); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("get unsupported task", "type", task->get_task_type()); + } + } + + if (OB_FAIL(ret)) { + err_handle_->handle_error(ret, "ObLogMinerResourceCollector exit unexpected\n"); + } + return ret; +} + +int ObLogMinerResourceCollector::push_task_(ObLogMinerRecyclableTask *task) +{ + int ret = OB_SUCCESS; + const int64_t DATA_OP_TIMEOUT = 1L * 1000 * 1000; + static int64_t seq_no = 0; + int64_t hash_val = ATOMIC_AAF(&seq_no, 1); + RETRY_FUNC(is_stoped(), *(static_cast(this)), push, task, hash_val, DATA_OP_TIMEOUT); + return ret; +} + +int ObLogMinerResourceCollector::handle_binlog_record_(ObLogMinerBR *br) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(br)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get null br in resource collector", K(br)); + } else { + br->destroy(); + ob_free(br); + } + return ret; +} + +int ObLogMinerResourceCollector::handle_logminer_record_(ObLogMinerRecord *logminer_rec) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(logminer_rec)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get null logminer_rec in resource collector", K(logminer_rec)); + } else if (OB_FAIL(data_manager_->release_log_miner_record(logminer_rec))) { + LOG_ERROR("failed to release log miner record", KPC(logminer_rec)); + } + return ret; +} + +int ObLogMinerResourceCollector::handle_batch_record_(ObLogMinerBatchRecord *batch_rec) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(batch_rec)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("get null batch record in resource collector", K(batch_rec)); + } else if (OB_FAIL(data_manager_->release_logminer_batch_record(batch_rec))) { + LOG_ERROR("failed to release batch record", KPC(batch_rec)); + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_resource_collector.h b/src/logservice/logminer/ob_log_miner_resource_collector.h new file mode 100644 index 0000000000..916158c66b --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_resource_collector.h @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_RESOURCE_COLLECTOR_H_ +#define OCEANBASE_LOG_MINER_RESOURCE_COLLECTOR_H_ + +#include "lib/thread/ob_multi_fixed_queue_thread.h" +#include "ob_log_miner_error_handler.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerBR; +class ObLogMinerRecord; +class ObLogMinerUndoTask; +class ObLogMinerBatchRecord; +class ObLogMinerRecyclableTask; +class ILogMinerBRProducer; +class ILogMinerDataManager; + +class ILogMinerResourceCollector { +public: + virtual int start() = 0; + virtual void stop() = 0; + virtual void wait() = 0; + virtual void destroy() = 0; + virtual int revert(ObLogMinerRecyclableTask *undo_task) = 0; + virtual int get_total_task_count(int64_t &task_count) = 0; +}; + +typedef common::ObMQThread<1, ILogMinerResourceCollector> ResourceCollectorThread; + +class ObLogMinerResourceCollector: public ILogMinerResourceCollector, ResourceCollectorThread +{ +public: + static const int64_t RC_THREAD_NUM; + static const int64_t RC_QUEUE_SIZE; +public: + virtual int start(); + virtual void stop(); + virtual void wait(); + virtual void destroy(); + virtual int revert(ObLogMinerRecyclableTask *undo_task); + virtual int get_total_task_count(int64_t &task_count); + +public: + virtual int handle(void *data, const int64_t thread_index, volatile bool &stop_flag); + +public: + ObLogMinerResourceCollector(); + ~ObLogMinerResourceCollector(); + int init(ILogMinerDataManager *data_manager, + ILogMinerErrorHandler *err_handle); + +private: + int push_task_(ObLogMinerRecyclableTask *task); + int handle_binlog_record_(ObLogMinerBR *br); + int handle_logminer_record_(ObLogMinerRecord *logminer_rec); + int handle_batch_record_(ObLogMinerBatchRecord *batch_rec); + +private: + bool is_inited_; + ILogMinerDataManager *data_manager_; + ILogMinerErrorHandler *err_handle_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_timezone_getter.cpp b/src/logservice/logminer/ob_log_miner_timezone_getter.cpp new file mode 100644 index 0000000000..52bf8319e6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_timezone_getter.cpp @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_timezone_getter.h" +#include "lib/string/ob_string.h" +namespace oceanbase +{ +namespace oblogminer +{ + +ObLogMinerTimeZoneGetter &ObLogMinerTimeZoneGetter::get_instance() +{ + static ObLogMinerTimeZoneGetter tz_getter_instance; + return tz_getter_instance; +} + +ObLogMinerTimeZoneGetter::ObLogMinerTimeZoneGetter(): + tz_info_() { } + +int ObLogMinerTimeZoneGetter::set_timezone(const char *timezone) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(tz_info_.set_timezone(ObString(timezone)))) { + LOG_ERROR("parse timezone failed", K(ret), KCSTRING(timezone)); + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_timezone_getter.h b/src/logservice/logminer/ob_log_miner_timezone_getter.h new file mode 100644 index 0000000000..54ea453c0c --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_timezone_getter.h @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_TIMEZONE_GETTER_H_ +#define OCEANBASE_LOG_MINER_TIMEZONE_GETTER_H_ + +#include "lib/timezone/ob_timezone_info.h" // ObTimeZoneInfo + +namespace oceanbase +{ +namespace oblogminer +{ +#define LOGMINER_TZ \ +::oceanbase::oblogminer::ObLogMinerTimeZoneGetter::get_instance() + +class ObLogMinerTimeZoneGetter { +public: + static ObLogMinerTimeZoneGetter &get_instance(); + + ObLogMinerTimeZoneGetter(); + + int set_timezone(const char *timezone); + + const ObTimeZoneInfo &get_tz_info() const { + return tz_info_; + } + +private: + ObTimeZoneInfo tz_info_; +}; +} +} + +#endif diff --git a/src/logservice/logminer/ob_log_miner_undo_task.cpp b/src/logservice/logminer/ob_log_miner_undo_task.cpp new file mode 100644 index 0000000000..e981fc55c6 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_undo_task.cpp @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_undo_task.h b/src/logservice/logminer/ob_log_miner_undo_task.h new file mode 100644 index 0000000000..2d842337af --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_undo_task.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_UNDO_TASK_H_ +#define OCEANBASE_LOG_UNDO_TASK_H_ + +#include "lib/string/ob_sql_string.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +class ObLogMinerRecord; + +class ObLogMinerUndoTask { +public: + ObLogMinerUndoTask(); + int init(ObLogMinerRecord *logminer_rec); + +private: + bool is_filtered_; + int64_t stmt_timestamp_; + common::ObSqlString *undo_stmt_; +}; +} +} + +#endif \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_utils.cpp b/src/logservice/logminer/ob_log_miner_utils.cpp new file mode 100644 index 0000000000..57d06de244 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_utils.cpp @@ -0,0 +1,458 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LOGMNR + +#include "ob_log_miner_utils.h" +#include "lib/ob_define.h" +#include "lib/function/ob_function.h" // ObFunction +#include + +#define APPEND_STR(buf, args...) \ + do {\ + if (OB_SUCC(ret) && OB_FAIL(buf.append(args))) { \ + LOG_ERROR("append str failed", "buf_len", buf.length(), "buf_cap", buf.capacity()); \ + }\ + } while(0) + +namespace oceanbase +{ +namespace oblogminer +{ + +// not very safe, str must end with '\0' +int logminer_str2ll(const char *str, char* &endptr, int64_t &num) +{ + int ret = OB_SUCCESS; + if (nullptr == str) { + ret = OB_INVALID_ARGUMENT; + } else { + int origin_errno = errno; + errno = 0; + num = strtoll(str, &endptr, 10); + if (0 != errno || endptr == str) { + ret = OB_INVALID_ARGUMENT; + } + errno = origin_errno; + } + + return ret; +} + +int logminer_str2ll(const char *str, int64_t &num) +{ + char *endptr = nullptr; + return logminer_str2ll(str, endptr, num); +} + +int write_keys(const KeyArray &key_arr, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + int64_t total_size = 0; + char *tmp_buffer = nullptr; + const char *keys_delimiter = "/"; + APPEND_STR(buffer, "\""); + ARRAY_FOREACH_N(key_arr, idx, key_num) { + const ObString &key = key_arr.at(idx); + APPEND_STR(buffer, key); + if (idx != key_num - 1) { + APPEND_STR(buffer, keys_delimiter); + } + } + APPEND_STR(buffer, "\""); + + return ret; +} + +int write_signed_number(const int64_t num, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + const int64_t MAX_NUM_LEN = 30; + char buf[MAX_NUM_LEN] = {0}; + int len = snprintf(buf, MAX_NUM_LEN, "%ld", num); + APPEND_STR(buffer, buf, len); + return ret; +} + +int write_unsigned_number(const uint64_t num, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + const int64_t MAX_NUM_LEN = 30; + char buf[MAX_NUM_LEN] = {0}; + int len = snprintf(buf, MAX_NUM_LEN, "%lu", num); + APPEND_STR(buffer, buf, len); + return ret; +} + +int write_string_no_escape(const ObString &str, common::ObStringBuffer &buffer) +{ + int ret = OB_SUCCESS; + APPEND_STR(buffer, "\""); + APPEND_STR(buffer, str); + APPEND_STR(buffer, "\""); + return ret; +} + +bool is_number(const char *str) +{ + bool bool_ret = true; + if (nullptr == str) { + return false; + } + for (int64_t i = 0; true == bool_ret; ++i) { + if ('\0' == str[i]) { + break; + } else if (!isdigit(str[i])) { + bool_ret = false; + } + } + return bool_ret; +} + +const char *empty_str_wrapper(const char *str) +{ + return nullptr == str ? "" : str; +} + +const char *record_type_to_str(const RecordType type) +{ + static const char *str = "UNKNOWN"; + + switch (type) + { + case EDELETE: + str = "DELETE"; + break; + + case EINSERT: + str = "INSERT"; + break; + + case EUPDATE: + str = "UPDATE"; + break; + + case EBEGIN: + str = "BEGIN"; + break; + + case ECOMMIT: + str = "COMMIT"; + break; + + case EDDL: + str = "DDL"; + break; + + default: + str = "NOT_SUPPORTED"; + break; + } + + return str; +} + +int64_t record_type_to_num(const RecordType type) +{ + int num = 0; + switch (type) + { + case EINSERT: + num = 1; + break; + + case EUPDATE: + num = 2; + break; + + case EDELETE: + num = 3; + break; + + case EDDL: + num = 4; + break; + + case EBEGIN: + num = 5; + break; + + case ECOMMIT: + num = 6; + break; + + default: + num = 0; + break; + } + + return num; + +} + +bool is_trans_end_record_type(const RecordType type) +{ + bool bret = false; + switch (type) { + case EDDL: + case ECOMMIT: + case HEARTBEAT: + bret = true; + break; + default: + bret = false; + break; + } + return bret; +} + +int parse_line(const char *key_str, + const char *buf, + const int64_t buf_len, + int64_t &pos, + int64_t &data) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(expect_token(buf, buf_len, pos, key_str))) { + LOG_ERROR("failed to get key_str from buf", K(buf_len), K(pos), K(key_str)); + } else if (OB_FAIL(expect_token(buf, buf_len, pos, "="))) { + LOG_ERROR("failed to get symbol '='", K(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(parse_int(buf, buf_len, pos, data))) { + LOG_ERROR("failed to parse int from buf", K(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(expect_token(buf, buf_len, pos, "\n"))) { + LOG_ERROR("failed to get token \\n", K(buf_len), K(pos)); + } + + return ret; +} + +int parse_int(const char *buf, const int64_t data_len, int64_t &pos, int64_t &data) +{ + int ret = OB_SUCCESS; + enum class ParseState { + BLANK_READ, + SYMBOL_READ, + DIGIT_READ + }; + ParseState state = ParseState::BLANK_READ; + bool positive = true; + bool done = false; + int64_t orig_pos = pos; + data = 0; + + while (pos < data_len && !done && OB_SUCC(ret)) { + char c = buf[pos]; + switch(state) { + case ParseState::BLANK_READ: { + if (isblank(c)) { + } else if (isdigit(c)) { + state = ParseState::DIGIT_READ; + data = todigit(c); + } else if ('+' == c) { + state = ParseState::SYMBOL_READ; + positive = true; + } else if ('-' == c) { + state = ParseState::SYMBOL_READ; + positive = false; + } else { + ret = OB_INVALID_DATA; + } + break; + } + + case ParseState::SYMBOL_READ: { + if (isdigit(c)) { + state = ParseState::DIGIT_READ; + if (positive) { + data = todigit(c); + } else { + data = -(todigit(c)); + } + } else { + ret = OB_INVALID_DATA; + } + break; + } + + case ParseState::DIGIT_READ: { + if (isdigit(c)) { + int digit = todigit(c); + if (positive) { + if (data > INT64_MAX/10) { + ret = OB_SIZE_OVERFLOW; + } else if (INT64_MAX/10 == data) { + if (digit > INT64_MAX % 10) { + ret = OB_SIZE_OVERFLOW; + } else { + data = data * 10 + digit; + } + } else { + data = data * 10 + digit; + } + } else { + if (data < INT64_MIN / 10) { + ret = OB_SIZE_OVERFLOW; + } else if (data == INT64_MIN/10) { + if (digit > -((INT64_MIN % 10 + 10) % 10 - 10)) { + ret = OB_SIZE_OVERFLOW; + } else { + data = data * 10 - digit; + } + } else { + data = data * 10 - digit; + } + } + } else { + done = true; + } + break; + } + } + + if (!done && OB_SUCC(ret)) { + pos++; + } + + } + + if (OB_FAIL(ret)) { + pos = orig_pos; + } + + + return ret; +} + +int parse_line(const char *key_str, + const char *buf, + const int64_t buf_len, + int64_t &pos, + ObIAllocator &alloc, + char *&value) +{ + int ret = OB_SUCCESS; + ObString tmp_str; + if (OB_FAIL(parse_line(key_str, buf, buf_len, pos, tmp_str))) { + LOG_ERROR("failed to parse line to tmp_str", K(buf_len), K(pos), K(key_str)); + } else if (OB_FAIL(ob_dup_cstring(alloc, tmp_str, value))) { + LOG_ERROR("failed to dup cstring to value", K(tmp_str)); + } + return ret; +} + +int parse_line(const char *key_str, + const char *buf, + const int64_t buf_len, + int64_t &pos, + ObString &value) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(expect_token(buf, buf_len, pos, key_str))) { + LOG_ERROR("failed to get key_str from buf", K(buf_len), K(pos), K(key_str)); + } else if (OB_FAIL(expect_token(buf, buf_len, pos, "="))) { + LOG_ERROR("failed to get symbol '='", K(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(parse_string(buf, buf_len, pos, value))) { + LOG_ERROR("failed to parse string from buf", K(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(expect_token(buf, buf_len, pos, "\n"))) { + LOG_ERROR("failed to get token \\n", K(buf_len), K(pos)); + } + return ret; +} + +int parse_string(const char *buf, const int64_t data_len, int64_t &pos, ObString &value) +{ + int ret = OB_SUCCESS; + const char *end_ptr = static_cast(memchr(buf + pos, '\n', data_len - pos)); + if (nullptr == end_ptr) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cannot find the end of line in buf", K(pos), K(data_len)); + } else { + const int64_t value_len = end_ptr - (buf + pos); + value.assign_ptr(buf + pos, value_len); + pos += value_len; + } + return ret; +} + +int expect_token(const char *buf, const int64_t data_len, int64_t &pos, const char *token) +{ + int ret = OB_SUCCESS; + + int64_t token_len = strlen(token); + + if (pos + token_len > data_len) { + ret = OB_SIZE_OVERFLOW; + } else if (0 != memcmp(buf + pos, token, token_len)) { + ret = OB_INVALID_DATA; + } else { + pos += token_len; + } + + return ret; +} + +int deep_copy_cstring(ObIAllocator &alloc, const char *src_str, char *&dst_str) +{ + int ret = OB_SUCCESS; + + if (nullptr != dst_str) { + ret = OB_ERR_UNEXPECTED; + } else if (nullptr == src_str) { + // do nothing + } else { + const int64_t src_str_len = strlen(src_str); + if (OB_ISNULL(dst_str = static_cast(alloc.alloc(src_str_len + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + MEMCPY(dst_str, src_str, src_str_len); + dst_str[src_str_len] = '\0'; + } + } + + return ret; +} + +int uint_to_bit(const uint64_t bit_uint, ObStringBuffer &bit_str) +{ + int ret = OB_SUCCESS; + const int bits = 64; + int i = bits-1; + if (0 == bit_uint) { + if (OB_FAIL(bit_str.append("0"))) { + LOG_ERROR("bit_str append failed", K(bit_str)); + } + } else { + // ignore leading zeroes + while (0 == (bit_uint & (1ull << i)) && 0 <= i) { + i--; + } + while (OB_SUCC(ret) && 0 <= i) { + if (OB_FAIL(bit_str.append((bit_uint & (1ull << i)) ? "1" : "0"))) { + LOG_ERROR("bit_str append failed", K(bit_str)); + } + i--; + } + } + return ret; +} + +int todigit(char c) +{ + return c - '0'; +} + + + +} +} + +#undef APPEND_STR \ No newline at end of file diff --git a/src/logservice/logminer/ob_log_miner_utils.h b/src/logservice/logminer/ob_log_miner_utils.h new file mode 100644 index 0000000000..66c8306bb0 --- /dev/null +++ b/src/logservice/logminer/ob_log_miner_utils.h @@ -0,0 +1,115 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_UTILS_H_ +#define OCEANBASE_LOG_MINER_UTILS_H_ + +#include "lib/alloc/alloc_struct.h" +#include "lib/allocator/ob_malloc.h" +#include "lib/container/ob_se_array.h" //ObSEArray +#include "lib/string/ob_fixed_length_string.h" +#include "lib/string/ob_string_buffer.h" +#include "libobcdc.h" +#ifndef OB_USE_DRCMSG +#include "ob_cdc_msg_convert.h" +#endif +#include +namespace oceanbase +{ +namespace oblogminer +{ +typedef common::ObFixedLengthString DbName; +typedef common::ObFixedLengthString TableName; +typedef common::ObFixedLengthString ColumnName; +typedef common::ObFixedLengthString TenantName; +typedef common::ObSEArray KeyArray; + +int logminer_str2ll(const char *str, char* &endptr, int64_t &num); +int logminer_str2ll(const char *str, int64_t &num); + +int write_keys(const KeyArray &key_arr, common::ObStringBuffer &buffer); +int write_signed_number(const int64_t num, common::ObStringBuffer &buffer); +int write_unsigned_number(const uint64_t num, common::ObStringBuffer &buffer); +int write_string_no_escape(const ObString &str, common::ObStringBuffer &buffer); + +bool is_number(const char *str); + +const char *empty_str_wrapper(const char *str); + +const char *record_type_to_str(const RecordType type); +int64_t record_type_to_num(const RecordType type); +int parse_line(const char *key_str, const char *buf, const int64_t buf_len, int64_t &pos, int64_t &data); +int parse_int(const char *buf, const int64_t data_len, int64_t &pos, int64_t &data); +// allocate memory for value, which is a standard cstring +int parse_line(const char *key_str, + const char *buf, + const int64_t buf_len, + int64_t &pos, + ObIAllocator &alloc, + char *&value); +// no memory alloc, direct assign contents in buffer to value +int parse_line(const char *key_str, + const char *buf, + const int64_t buf_len, + int64_t &pos, + ObString &value); +// int parse_string(const char *buf, const int64_t data_len, int64_t &pos, ObIAllocator &alloc, char *&value); +int parse_string(const char *buf, const int64_t data_len, int64_t &pos, ObString &value); +int expect_token(const char *buf, const int64_t data_len, int64_t &pos, const char *token); + +int deep_copy_cstring(ObIAllocator &alloc, const char *src_str, char *&dst_str); + +bool is_trans_end_record_type(const RecordType type); + +int uint_to_bit(const uint64_t bit_val, ObStringBuffer &str_val); + +int todigit(char c); + +template +int init_component(Base *&ptr, Args&& ...arg) +{ + int ret = OB_SUCCESS; + lib::ObMemAttr attr(OB_SERVER_TENANT_ID, "CompnentAlloc"); + Derived *tmp_var = OB_NEW(Derived, attr); + const char *derived_name = typeid(Derived).name(); + const char *base_name = typeid(Base).name(); + if (OB_ISNULL(tmp_var)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOGMNR_LOG(ERROR, "allocate memory for component failed", K(tmp_var), + K(derived_name), K(base_name)); + } else if (OB_FAIL(tmp_var->init(arg...))) { + LOGMNR_LOG(ERROR, "init tmp_var failed", K(tmp_var), + K(derived_name), K(base_name)); + } else { + ptr = tmp_var; + LOGMNR_LOG(INFO, "init component finished", K(derived_name), K(base_name)); + } + return ret; +} + +template +void destroy_component(Base *&ptr) +{ + if (OB_NOT_NULL(ptr)) { + const char *derived_name = typeid(Derived).name(); + const char *base_name = typeid(Base).name(); + Derived *tmp_ptr = static_cast(ptr); + tmp_ptr->destroy(); + OB_DELETE(Derived, "ComponentFree", tmp_ptr); + ptr = nullptr; + LOGMNR_LOG(INFO, "finished to destroy component", K(derived_name), K(base_name)); + } +} +} +} + +#endif \ No newline at end of file diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 040a758694..1444b45047 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -32,3 +32,4 @@ add_subdirectory(share) add_subdirectory(rootserver) add_subdirectory(tools) add_subdirectory(data_dictionary) +add_subdirectory(logminer) diff --git a/unittest/logminer/CMakeLists.txt b/unittest/logminer/CMakeLists.txt new file mode 100644 index 0000000000..f15e579124 --- /dev/null +++ b/unittest/logminer/CMakeLists.txt @@ -0,0 +1,27 @@ +set(UTIL_FILE_NAME ob_log_miner_test_utils.cpp) + +function(logminer_unittest case) + if(ARGC EQUAL 1) + add_executable(${case} ${case}.cpp ${UTIL_FILE_NAME}) + else() + add_executable(${ARGV} ${UTIL_FILE_NAME}) + endif() + if (case MATCHES "^test_.*") + add_test(${case} ${case}) + set_tests_properties(${case} PROPERTIES TIMEOUT 300) + endif() + target_link_libraries(${case} PRIVATE oblogminer_obj_dev gtest gmock) + target_include_directories(${case} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/unittest ${CMAKE_SOURCE_DIR}/deps/oblib/unittest) +endfunction() + +logminer_unittest(test_ob_log_miner_args) +logminer_unittest(test_ob_log_miner_br_filter) +logminer_unittest(test_ob_log_miner_record) +logminer_unittest(test_ob_log_miner_record_converter) +logminer_unittest(test_ob_log_miner_utils) +logminer_unittest(test_ob_log_miner_analyzer_checkpoint) +logminer_unittest(test_ob_log_miner_progress_range) +logminer_unittest(test_ob_log_miner_file_meta) +logminer_unittest(test_ob_log_miner_file_index) +logminer_unittest(test_ob_log_miner_file_manager) \ No newline at end of file diff --git a/unittest/logminer/ob_log_miner_test_utils.cpp b/unittest/logminer/ob_log_miner_test_utils.cpp new file mode 100644 index 0000000000..6d822e1bae --- /dev/null +++ b/unittest/logminer/ob_log_miner_test_utils.cpp @@ -0,0 +1,186 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_binlog_record.h" +#include "ob_log_miner_br.h" +#include "ob_log_miner_test_utils.h" +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +ObLogMinerBR *v_build_logminer_br(binlogBuf *new_bufs, + binlogBuf *old_bufs, + RecordType type, + lib::Worker::CompatMode compat_mode, + const char *db_name, + const char *table_name, + const char *encoding, + const int arg_count, va_list *ap) +{ + ObLogMinerBR *br = new ObLogMinerBR; + EXPECT_NE(nullptr, br); + libobcdc::ObLogBR *cdc_br = new libobcdc::ObLogBR; + cdc_br->init_data(EINSERT, 0, 1, 0, ObString(), ObString(), ObString(), + 100, 100); + ICDCRecord *rec = DRCMessageFactory::createBinlogRecord(); + rec->setUserData(cdc_br); + EXPECT_NE(nullptr, rec); + ITableMeta *tab_meta = DRCMessageFactory::createTableMeta(); + EXPECT_NE(nullptr, tab_meta); + IDBMeta *db_meta = DRCMessageFactory::createDBMeta(); + EXPECT_NE(nullptr, db_meta); + rec->setNewColumn(new_bufs, arg_count/3); + rec->setOldColumn(old_bufs, arg_count/3); + rec->setRecordType(type); + rec->setDbname(db_name); + rec->setTbname(table_name); + db_meta->setName(db_name); + tab_meta->setPKs(""); + tab_meta->setPkinfo(""); + tab_meta->setUKs(""); + tab_meta->setUkinfo(""); + tab_meta->setName(table_name); + tab_meta->setDBMeta(db_meta); + IColMeta *col_meta = nullptr; + const char *next = nullptr; + int64_t arg_idx = 0; + for (int i = 0; i < arg_count; i++) { + const char *col_arg = nullptr; + int len = 0; + int data_type = 0; + if (3 > i%4) { + col_arg = va_arg(*ap, const char *); + len = col_arg == nullptr ? 0 : strlen(col_arg); + } else { + data_type = va_arg(*ap, int); + } + if (0 == i%4) { // column name + col_meta = DRCMessageFactory::createColMeta(); + EXPECT_NE(col_meta, nullptr); + col_meta->setName(col_arg); + col_meta->setEncoding(encoding); + tab_meta->append(col_arg, col_meta); + } else if (1 == i%4 && EDELETE != type) { // new value, EDELETE hasn't new value + rec->putNew(col_arg, len); + } else if (2 == i%4 && EINSERT != type) { // old value, EINSERT hasn't old value + rec->putOld(col_arg, len); + } else if (3 == i%4) { // column data type + col_meta->setType(data_type); + } + } + rec->setTableMeta(tab_meta); + br->init(nullptr, rec, compat_mode, 0, 1, 0); + return br; +} + +ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs, + binlogBuf *old_bufs, + RecordType type, + lib::Worker::CompatMode compat_mode, + const char *db_name, + const char *table_name, + const char *encoding, + const int arg_count, ...) +{ + va_list ap; + ObLogMinerBR *br = nullptr; + va_start(ap, arg_count); + br = v_build_logminer_br(new_bufs, old_bufs, type, + compat_mode, db_name, table_name, encoding, arg_count, &ap); + va_end(ap); + return br; +} + +ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs, + binlogBuf *old_bufs, + RecordType type, + lib::Worker::CompatMode compat_mode, + const char *db_name, + const char *table_name, + const int arg_count, ...) +{ + va_list ap; + ObLogMinerBR *br = nullptr; + va_start(ap, arg_count); + br = v_build_logminer_br(new_bufs, old_bufs, type, + compat_mode, db_name, table_name, "utf8mb4", arg_count, &ap); + va_end(ap); + return br; +} + +void destroy_miner_br(ObLogMinerBR *&br) { + ICDCRecord *cdc_br = br->get_br(); + IDBMeta *db_meta = cdc_br->getDBMeta(); + ITableMeta *tbl_meta = cdc_br->getTableMeta(); + delete static_cast(cdc_br->getUserData()); + DRCMessageFactory::destroy(db_meta); + DRCMessageFactory::destroy(tbl_meta); + DRCMessageFactory::destroy(cdc_br); + delete br; + br = nullptr; +} + +ObLogMinerRecord *build_logminer_record(ObIAllocator &alloc, + lib::Worker::CompatMode compat_mode, + uint64_t tenant_id, + int64_t orig_cluster_id, + const char *tenant_name, + const char *database_name, + const char *tbl_name, + int64_t trans_id, + const char* const * pks, + const int64_t pk_cnt, + const char* const * uks, + const int64_t uk_cnt, + const char* row_unique_id, + RecordType record_type, + int64_t commit_ts_us, + const char * redo_stmt, + const char * undo_stmt) +{ + void *ptr = ob_malloc(sizeof(ObLogMinerRecord), "testminer_rec"); + ObLogMinerRecord *rec = new (ptr) (ObLogMinerRecord); + rec->set_allocator(&alloc); + + rec->compat_mode_ = compat_mode; + rec->tenant_id_ = tenant_id; + rec->orig_cluster_id_ = orig_cluster_id; + rec->tenant_name_.assign(tenant_name); + rec->database_name_.assign(database_name); + rec->table_name_.assign(tbl_name); + rec->trans_id_ = trans_id; + for (int i = 0; i < pk_cnt; i++) { + rec->primary_keys_.push_back(pks[i]); + } + for (int i = 0; i < uk_cnt; i++) { + rec->unique_keys_.push_back(uks[i]); + } + rec->row_unique_id_ = row_unique_id; + rec->record_type_ = record_type; + rec->commit_scn_.convert_from_ts(commit_ts_us); + rec->redo_stmt_.append(redo_stmt); + rec->undo_stmt_.append(undo_stmt); + + return rec; +} + +void destroy_miner_record(ObLogMinerRecord *&rec) +{ + rec->redo_stmt_.reset(); + rec->undo_stmt_.reset(); + ob_free(rec); +} +} +} \ No newline at end of file diff --git a/unittest/logminer/ob_log_miner_test_utils.h b/unittest/logminer/ob_log_miner_test_utils.h new file mode 100644 index 0000000000..2fb8ac5a5f --- /dev/null +++ b/unittest/logminer/ob_log_miner_test_utils.h @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_MINER_UNITTEST_UTILS_H_ +#define OCEANBASE_LOG_MINER_UNITTEST_UTILS_H_ + +#include "ob_log_miner_br.h" +#include "rpc/obmysql/ob_mysql_global.h" +#include "gtest/gtest.h" +#define private public +#include "ob_log_miner_record.h" +#undef private +namespace oceanbase +{ +namespace oblogminer +{ +ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs, + binlogBuf *old_bufs, + RecordType type, + lib::Worker::CompatMode compat_mode, + const char *db_name, + const char *table_name, + const int arg_count, ...); + +ObLogMinerBR *build_logminer_br(binlogBuf *new_bufs, + binlogBuf *old_bufs, + RecordType type, + lib::Worker::CompatMode compat_mode, + const char *db_name, + const char *table_name, + const char *encoding, + const int arg_count, ...); + +void destroy_miner_br(ObLogMinerBR *&br); + +ObLogMinerRecord *build_logminer_record(ObIAllocator &alloc, + lib::Worker::CompatMode compat_mode, + uint64_t tenant_id, + int64_t orig_cluster_id, + const char *tenant_name, + const char *database_name, + const char *tbl_name, + int64_t trans_id, + const char* const * pks, + const int64_t pk_cnt, + const char* const * uks, + const int64_t uk_cnt, + const char* row_unique_id, + RecordType record_type, + int64_t commit_ts_us, + const char * redo_stmt, + const char * undo_stmt); + +void destroy_miner_record(ObLogMinerRecord *&rec); + +} +} + +#endif \ No newline at end of file diff --git a/unittest/logminer/test_ob_log_miner_analyzer_checkpoint.cpp b/unittest/logminer/test_ob_log_miner_analyzer_checkpoint.cpp new file mode 100644 index 0000000000..61e9f5ef3f --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_analyzer_checkpoint.cpp @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_miner_analyzer_checkpoint.h" +#include "lib/oblog/ob_log.h" +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_log_miner_analyzer_ckpt, SerializeFunc) +{ + ObLogMinerCheckpoint ckpt, ckpt1; + int64_t pos = 0; + const char *buf1 = "PROGRESS=1\nCUR_FILE_ID=3\nMAX_FILE_ID=2\n"; + char tmp_buf[1000]; + EXPECT_EQ(OB_SUCCESS, ckpt.deserialize(buf1, strlen(buf1), pos)); + EXPECT_EQ(ckpt.progress_, 1); + EXPECT_EQ(ckpt.cur_file_id_, 3); + EXPECT_EQ(ckpt.max_file_id_, 2); + pos = 0; + EXPECT_EQ(OB_SUCCESS, ckpt.serialize(tmp_buf, 1000, pos)); + tmp_buf[pos+1] = '\0'; + EXPECT_STREQ(buf1, tmp_buf); + pos = 0; + const char *buf2 = "PROGRESS=a\nCUR_FILE_ID=3\nMAX_FILE_ID=2\n"; + EXPECT_EQ(OB_INVALID_DATA, ckpt.deserialize(buf2, strlen(buf2), pos)); + pos = 0; + const char *buf3 = "PROGRESS=1\nCUR_FILE_ID=3MAX_FILE_ID=2\n"; + EXPECT_EQ(OB_INVALID_DATA, ckpt.deserialize(buf3, strlen(buf3), pos)); + pos = 0; + + for (int i = 0; i < 10000; i++) { + int64_t pos1 = 0; + pos = 0; + ckpt.cur_file_id_ = rand(); + ckpt.max_file_id_ = rand(); + ckpt.progress_ = rand(); + EXPECT_EQ(OB_SUCCESS, ckpt.serialize(tmp_buf, 1000, pos)); + EXPECT_EQ(pos, ckpt.get_serialize_size()); + EXPECT_EQ(OB_SUCCESS, ckpt1.deserialize(tmp_buf, pos, pos1)); + EXPECT_EQ(ckpt, ckpt1); + } +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_analyzer_checkpoint.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_analyzer_checkpoint.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_args.cpp b/unittest/logminer/test_ob_log_miner_args.cpp new file mode 100644 index 0000000000..37fb9e0633 --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_args.cpp @@ -0,0 +1,1083 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "gtest/gtest.h" + +#include "lib/allocator/page_arena.h" +#include "lib/string/ob_string_buffer.h" +#include "ob_log_miner_args.h" +#include "lib/oblog/ob_log.h" + + +using namespace oceanbase; + +namespace oceanbase +{ +namespace oblogminer +{ + +struct CmdArgs { + explicit CmdArgs(ObIAllocator &alloc): + allocator(alloc), + argc(0) {} + + ~CmdArgs() { + reset(); + } + + void build_cmd_args(const char* prog_name, ...) { + reset(); + + va_list ap; + va_start(ap, prog_name); + + argv[argc] = (char*)allocator.alloc(strlen(prog_name)+1); + strncpy(argv[argc], prog_name, strlen(prog_name)); + argv[argc][strlen(prog_name)] = '\0'; + argc++; + + const char* next = nullptr; + while ((next = va_arg(ap, const char*)) != nullptr && argc < 100) { + argv[argc] = (char*)allocator.alloc(strlen(next)+1); + strncpy(argv[argc], next, strlen(next)); + argv[argc][strlen(next)] = '\0'; + argc++; + } + + va_end(ap); + } + + void reset() { + for (int i = 0; i < argc; i++) { + allocator.free(argv[i]); + argv[i] = nullptr; + } + argc = 0; + optind = 1; + } + + ObIAllocator &allocator; + int argc; + char *argv[100]; +}; + + + +TEST(ObLogMinerArgs, test_log_miner_args) +{ + ObMalloc allocator("TestMnrCmdArgs"); + CmdArgs *args = new CmdArgs(allocator); + ObLogMinerCmdArgs miner_cmd_args; + ObLogMinerArgs miner_args; + EXPECT_NE(nullptr, args); + + // test commands for analysis mode + // set analyze archvie + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // set analyze online + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // unknown arguments, unexpected + args->build_cmd_args("oblogminer", + "-x", "unknown_arg", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // unknown mode, OB_INVALID_ARGUMENT + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "unknown_mode", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // both archive dest and online are set + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack online params + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // archive_dest and part online arguments are given + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-c", "cluster_url", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack output + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack table-list, default not filter any table + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack start time + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // recovery path is set in analysis mode + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-r", "recovery_path", + nullptr); + EXPECT_EQ(OB_ERR_UNEXPECTED, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // source_db1.table is set in analysis mode + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-S", "db1.table1", + nullptr); + EXPECT_EQ(OB_ERR_UNEXPECTED, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // target table is set in analysis mode + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-t", "db1.tmp_table1", + nullptr); + EXPECT_EQ(OB_ERR_UNEXPECTED, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + + // test commands for analysis mode + // args are correctly set + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack cluster_url + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-u", "username@tenant#cluster", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack recovery_path + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack source table + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-r", "recovery_path", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // lack output_dest + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + // archvie_dest is set in flashback mode + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-a", "archvie_dest", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_ERR_UNEXPECTED, miner_cmd_args.init(args->argc, args->argv)); + + // table_list is set in flashback mode + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-l", "table-list", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_ERR_UNEXPECTED, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + + // verbose + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-v", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + + // timezone + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-z", "+8:00", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + + // timezone + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-z", "-8:00", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + + // unspecified end time + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-o", "output_dest", + "-z", "+0:00", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_cmd_args.init(args->argc, args->argv)); + miner_cmd_args.reset(); + delete args; +} + +TEST(ObLogMinerArgs, test_log_miner_analyzer_flashbacker_args) { + ObMalloc allocator("TestMnrCmdArgs"); + CmdArgs *args = new CmdArgs(allocator); + ObLogMinerCmdArgs miner_args; + AnalyzerArgs analyzer_args; + FlashbackerArgs flashbacker_args; + EXPECT_NE(nullptr, args); + + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + EXPECT_EQ(OB_ERR_UNEXPECTED, flashbacker_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + flashbacker_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "aaaaa", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_DATE_FORMAT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + // incorrect time range + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "22222", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-c", "cluster_addr", + "-u", "user_name@tenant", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "22221", + "-e", "22222", + "-o", "output_progress", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, flashbacker_args.init(miner_args)); + EXPECT_EQ(OB_ERR_UNEXPECTED, analyzer_args.init(miner_args)); + miner_args.reset(); + flashbacker_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-20 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-20 16:35:20", + "-o", "output_dest", + "-z", "-8:00", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-20 16:35:20", + "-o", "output_dest", + "-z", "xxx", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_ERR_UNKNOWN_TIME_ZONE, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-20 16:35:20", + "-e", "2023-12-20 16:35:20", + "-o", "output_dest", + "-z", "+8:00", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-20 16:35:20", + "-e", "2023-12-20 16:35:20.000001", + "-o", "output_dest", + "-z", "+8:00", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-20 16:35:20", + "-e", "2023-12-20 16:35:20.000001", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-32 16:35:20", + "-e", "2023-12-32 16:35:20.000001", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_DATE_VALUE, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + // check user-name + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + // check password + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + // table list check + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "tenant.db1.table1", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.*.*", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "fake.db1.*", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*..", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.*|tenant.db2.*", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "*.db1.*|fake.db2.*", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-O", "insert|delete", + "-m", "analysis", + "-l", "tenant.db1.*", + "-s", "2023-12-31 16:35:20", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + miner_args.reset(); + analyzer_args.reset(); + + // incorrect time range + args->build_cmd_args("oblogminer", + "-m", "flashback", + "-c", "cluster_addr", + "-u", "user_name", + "-p", "password", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.tmp_table1", + "-s", "22222", + "-e", "22222", + "-o", "output_progress", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_INVALID_ARGUMENT, flashbacker_args.init(miner_args)); + miner_args.reset(); + flashbacker_args.reset(); + if (nullptr != args) { + delete args; + } +} + +TEST(ObLogMinerArgs, test_ob_log_miner_args) +{ + ObMalloc allocator("TestMnrCmdArgs"); + CmdArgs *args = new CmdArgs(allocator); + ObLogMinerArgs miner_args; + EXPECT_NE(nullptr, args); + + // test commands for analysis mode + // set analyze archvie + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + OB_LOGGER.set_log_level("DEBUG"); + OB_LOGGER.set_mod_log_levels("ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN"); + // set analyze online + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "flashback", + "-r", "recovery_path", + "-S", "db1.table1", + "-t", "db1.table1_tmp", + "-s", "11111", + "-e", "22222", + "-o", "progress_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "flashback", + "-S", "db1.table1", + "-t", "db1.table1_tmp", + "-s", "23333", + "-e", "22222", + "-o", "progress_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + // unknown arguments, unexpected + args->build_cmd_args("oblogminer", + "-x", "unknown_arg", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-r", "recovery_path", + nullptr); + EXPECT_EQ(OB_ERR_UNEXPECTED, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + + args->build_cmd_args("oblogminer", + "-c", "cluster_url", + "-u", "username@tenant#cluster", + "-p", "password", + "-m", "flashback", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + "-r", "recovery_path", + nullptr); + EXPECT_EQ(OB_INVALID_ARGUMENT, miner_args.init(args->argc, args->argv)); + miner_args.reset(); + if (nullptr != args) { + delete args; + } +} + +TEST(ObLogMinerArgs, test_log_miner_analyzer_args_serialize) +{ + ObMalloc allocator("TestMnrCmdArgs"); + CmdArgs *args = new CmdArgs(allocator); + ObLogMinerCmdArgs miner_args; + AnalyzerArgs analyzer_args; + EXPECT_NE(nullptr, args); + + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.init(miner_args)); + int64_t pos = 0; + const char *expect_buf = "cluster_addr=\nuser_name=\npassword=\narchive_dest=archive_dest\ntable_list=*.db1.table1\n" + "column_cond=\noperations=insert|delete|update\noutput_dest=output_dest\nlog_level=\n" + "start_time_us=11111\nend_time_us=22222\ntimezone=+8:00\nrecord_format=CSV\n"; + char tmp_buf[1000] = {0}; + EXPECT_EQ(OB_SUCCESS, analyzer_args.serialize(tmp_buf, 1000, pos)); + EXPECT_STREQ(tmp_buf, expect_buf); + analyzer_args.reset(); + pos = 0; + EXPECT_EQ(OB_SUCCESS, analyzer_args.deserialize(expect_buf, strlen(expect_buf), pos)); + EXPECT_STREQ(analyzer_args.archive_dest_, "archive_dest"); + EXPECT_STREQ(analyzer_args.table_list_, "*.db1.table1"); + EXPECT_EQ(analyzer_args.start_time_us_, 11111); + EXPECT_EQ(analyzer_args.end_time_us_, 22222); + EXPECT_STREQ(analyzer_args.output_dst_, "output_dest"); + pos = 0; + memset(tmp_buf, 0, sizeof(tmp_buf)); + EXPECT_EQ(OB_SUCCESS, analyzer_args.serialize(tmp_buf, 1000, pos)); + tmp_buf[pos+1] = '\0'; + EXPECT_STREQ(expect_buf, tmp_buf); + pos = 0; + const char *buf2 = "cluster_addr=\nuser_name=\n"; + analyzer_args.reset(); + EXPECT_EQ(OB_SIZE_OVERFLOW, analyzer_args.deserialize(buf2, strlen(buf2), pos)); + pos = 0; + const char *buf3 = "cluster_addr=\nuser_name=\npassword=\narchive_dest=archive_dest\ntable_list=db1.table1\n" + "operations=insert|delete|update\noutput_dest=output_dest\nlog_level=ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN\n" + "start_time_us=11a11\nend_time_us=22222\ntimezone=+8:00\nrecord_format=CSV\n"; + EXPECT_EQ(OB_INVALID_DATA, analyzer_args.deserialize(buf3, strlen(buf3), pos)); + analyzer_args.reset(); + miner_args.reset(); + if (nullptr != args) { + delete args; + } +} + +TEST(ObLogMinerArgs, test_log_miner_args_serialize) +{ + ObMalloc allocator("TestMnrArgs"); + CmdArgs *args = new CmdArgs(allocator); + ObLogMinerArgs miner_args; + EXPECT_NE(nullptr, args); + + // test commands for analysis mode + // set analyze archvie + args->build_cmd_args("oblogminer", + "-a", "archive_dest", + "-m", "analysis", + "-l", "*.db1.table1", + "-s", "11111", + "-e", "22222", + "-o", "output_dest", + nullptr); + EXPECT_EQ(OB_SUCCESS, miner_args.init(args->argc, args->argv)); + int64_t pos = 0; + const char *expect_buf = "mode=ANALYSIS\ncluster_addr=\nuser_name=\npassword=\n" + "archive_dest=archive_dest\ntable_list=*.db1.table1\ncolumn_cond=\noperations=insert|delete|update\n" + "output_dest=output_dest\nlog_level=ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN\nstart_time_us=11111\n" + "end_time_us=22222\ntimezone=+8:00\nrecord_format=CSV\n"; + char tmp_buf[1000] = {0}; + EXPECT_EQ(OB_SUCCESS, miner_args.serialize(tmp_buf, 1000, pos)); + EXPECT_STREQ(tmp_buf, expect_buf); + pos = 0; + EXPECT_EQ(OB_SUCCESS, miner_args.deserialize(expect_buf, strlen(expect_buf), pos)); + EXPECT_EQ(miner_args.mode_, LogMinerMode::ANALYSIS); + EXPECT_STREQ(miner_args.analyzer_args_.archive_dest_, "archive_dest"); + EXPECT_STREQ(miner_args.analyzer_args_.table_list_, "*.db1.table1"); + EXPECT_EQ(miner_args.analyzer_args_.start_time_us_, 11111); + EXPECT_EQ(miner_args.analyzer_args_.end_time_us_, 22222); + EXPECT_STREQ(miner_args.analyzer_args_.output_dst_, "output_dest"); + pos = 0; + memset(tmp_buf, 0, sizeof(tmp_buf)); + EXPECT_EQ(OB_SUCCESS, miner_args.serialize(tmp_buf, 1000, pos)); + tmp_buf[pos+1] = '\0'; + EXPECT_STREQ(expect_buf, tmp_buf); + + if (nullptr != args) { + delete args; + } +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_log_miner_args.log"); + ObLogger &logger = ObLogger::get_logger(); + logger.set_file_name("test_log_miner_args.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_br_filter.cpp b/unittest/logminer/test_ob_log_miner_br_filter.cpp new file mode 100644 index 0000000000..e7bd6dca00 --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_br_filter.cpp @@ -0,0 +1,520 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "gtest/gtest.h" +#include "lib/oblog/ob_log.h" +#include "ob_log_miner_br_filter.h" +#include "ob_log_miner_filter_condition.h" +#include "ob_log_miner_br.h" +#include "ob_log_miner_test_utils.h" + + +using namespace oceanbase; + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(ob_log_miner_br_filter, ColumnBRFilterPlugin) +{ + ObArenaAllocator arena_alloc("FilterTest"); + ColumnBRFilterPlugin col_filter(&arena_alloc); + ObLogMinerBR *br = nullptr; + bool need_filter = false; + const int buf_cnt = 10; + binlogBuf *new_buf = static_cast(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt)); + binlogBuf *old_buf = static_cast(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt)); + for (int i = 0; i < 10; i++) { + new_buf[i].buf = static_cast(arena_alloc.alloc(1024)); + new_buf[i].buf_size = 1024; + new_buf[i].buf_used_size = 0; + old_buf[i].buf = static_cast(arena_alloc.alloc(1024)); + old_buf[i].buf_size = 1024; + old_buf[i].buf_used_size = 0; + } + EXPECT_EQ(OB_SUCCESS, col_filter.init("[]")); + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "val1", "val2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"table_name\":\"aaa\", \ + \"table_name\":\"aaa\"}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"tenant_name\":\"aaa\", \ + \"table_name\":\"aaa\"}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"aaa\", \ + \"database_name\":\"aaa\"}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"t1.aaa\", \ + \"table_name\":\"aaa\"}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"t1.aaa\", \ + \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": 3.1415926}]}]")); + col_filter.destroy(); + EXPECT_EQ(OB_SUCCESS, col_filter.init("[{\"database_name\":\"t1.aaa\", \ + \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"}]}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[{\"database_name\":\"t1.aaa\", \ + \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"}]")); + col_filter.destroy(); + EXPECT_EQ(OB_SUCCESS, col_filter.init("[{\"database_name\":\"t1.aaa\", \ + \"table_name\":\"aaa\", \"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"}, {\"col3\":null}]}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[\"table_name\":\"aaa\", " + "\"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"},{\"col3\":null}]}]")); + col_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("[\"column_cond\":[{\"col1\": \"3.1415926\",\"col2\":\"abcde\"},{\"col3\":null}]}]")); + col_filter.destroy(); + + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("db1.t1|db2.t2|db3.t*")); + col_filter.destroy(); + + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("db1.*|db2")); + col_filter.destroy(); + + EXPECT_EQ(OB_SUCCESS, col_filter.init("[" + "{\"database_name\":\"db1\",\"table_name\":\"table1\"," + "\"column_cond\":[{\"col1\":\"val1\"},{\"col2\":\"val2\"}]" + "}" + "]")); + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "val1", "val2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col2", "val3", "val2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col2", "val3", "val3", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), "col1", nullptr, "val1", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db2", "table1", 8, "col2", "val3", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col1", "val1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table2", 8, "col2", "val3", "val3", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col1", nullptr, "val1", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col2", "val3", "val3", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col3", nullptr, "val4", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + br = build_logminer_br(new_buf, old_buf, EDDL, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table2", 0); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + col_filter.destroy(); + + EXPECT_EQ(OB_INVALID_ARGUMENT, col_filter.init("db*.t1|*_prod.*_history|db3.t*")); + col_filter.destroy(); + + EXPECT_EQ(OB_SUCCESS, col_filter.init( + "[{\"database_name\":\"db1\", \"table_name\":\"table1\"," + "\"column_cond\":[{\"col1\":\"val1\", \"col2\":\"val2\"}]}]")); + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db2", "non_spec_tbl", 3, "non_spec_col", "non_spec_val", nullptr); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db0", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db3", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db3", "tmp_tbl", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val2", "val2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val1", "val3", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val1", "val1", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val1", "val3", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val10", "val100", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val2", "val2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val1", "val2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val1", "val2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", nullptr, "val1", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", nullptr, "val2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 12, "col1", "val10", "val1", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val20", "val2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col3", "val3", "val30", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + col_filter.destroy(); + + // null value filter + EXPECT_EQ(OB_SUCCESS, col_filter.init( + "[{\"database_name\":\"db1\", \"table_name\":\"table1\"," + "\"column_cond\":[{\"col1\":null}]}]")); + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db2", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE,lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "non_spec_val", "xx", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", nullptr, nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", nullptr, nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", nullptr, "xx", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + col_filter.destroy(); + + // multi column filter with null + EXPECT_EQ(OB_SUCCESS, col_filter.init( + "[{\"database_name\":\"db1\", \"table_name\":\"table1\"," + "\"column_cond\":[{\"col1\":\"val1\", \"col2\":\"val2\"}, {\"col3\":null}]}]")); + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db2", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT,lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col3", nullptr, nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val1", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val2", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "val1", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + col_filter.destroy(); + + + // multi column filter + EXPECT_EQ(OB_SUCCESS, col_filter.init( + "[{\"database_name\":\"db1\", \"table_name\":\"table1\"," + "\"column_cond\":[{\"col1\":\"val1\", \"col2\":\"val2\"}, {\"col3\":\"val3\", \"col4\":\"val4\"}]}]")); + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db2", "non_spec_tbl", 4, "non_spec_col", "non_spec_val", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "val1", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val1", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val2", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 8, "col1", "val1", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col3", "val3", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, col_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + col_filter.destroy(); +} + +TEST(ob_log_miner_br_filter, OperationBRFilterPlugin) +{ + OperationBRFilterPlugin op_filter; + ObArenaAllocator arena_alloc("OpFilterTest"); + const int64_t buf_cnt = 10; + ObLogMinerBR *br = nullptr; + bool need_filter = false; + binlogBuf *new_buf = static_cast(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt)); + binlogBuf *old_buf = static_cast(arena_alloc.alloc(sizeof(binlogBuf) * buf_cnt)); + for (int i = 0; i < 10; i++) { + new_buf[i].buf = static_cast(arena_alloc.alloc(1024)); + new_buf[i].buf_size = 1024; + new_buf[i].buf_used_size = 0; + old_buf[i].buf = static_cast(arena_alloc.alloc(1024)); + old_buf[i].buf_size = 1024; + old_buf[i].buf_used_size = 0; + } + EXPECT_EQ(OB_INVALID_ARGUMENT, op_filter.init("aaa|bbb")); + op_filter.destroy(); + EXPECT_EQ(OB_SUCCESS, op_filter.init("Insert")); + op_filter.destroy(); + EXPECT_EQ(OB_INVALID_ARGUMENT, op_filter.init("lnsert")); + op_filter.destroy(); + + EXPECT_EQ(OB_ERR_UNEXPECTED, op_filter.init(nullptr)); + op_filter.destroy(); + + + EXPECT_EQ(OB_SUCCESS, op_filter.init("insert")); + + br = build_logminer_br(new_buf, old_buf, EBEGIN, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, HEARTBEAT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + op_filter.destroy(); + + EXPECT_EQ(OB_SUCCESS, op_filter.init("insert|update")); + + br = build_logminer_br(new_buf, old_buf, HEARTBEAT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + op_filter.destroy(); + + EXPECT_EQ(OB_SUCCESS, op_filter.init("insert|update|delete")); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", "val10", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 12, "col1", "val10", "val1", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col2", "val20", "val2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "col3", "val3", "val30", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 4, "col1", nullptr, "val1", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EBEGIN, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, ECOMMIT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, HEARTBEAT, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EBEGIN, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(false, need_filter); + destroy_miner_br(br); + + br = build_logminer_br(new_buf, old_buf, EREPLACE, lib::Worker::CompatMode::MYSQL, + "tenant1.db1", "table1", 0); + EXPECT_EQ(OB_SUCCESS, op_filter.filter(*br, need_filter)); + EXPECT_EQ(true, need_filter); + destroy_miner_br(br); + + op_filter.destroy(); +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_filter.log"); + ObLogger &logger = ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_filter.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_file_index.cpp b/unittest/logminer/test_ob_log_miner_file_index.cpp new file mode 100644 index 0000000000..c67607c93e --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_file_index.cpp @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define private public +#include "ob_log_miner_file_index.h" +#undef private +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +static bool operator==(const FileIndex &lhs, const FileIndex &rhs) +{ + bool bret = true; + + if (lhs.index_file_len_ != rhs.index_file_len_) { + bret = false; + } else if (lhs.index_array_.count() != rhs.index_array_.count()) { + bret = false; + } else { + for (int64_t i = 0; bret && i < lhs.index_array_.count(); i++) { + const FileIndexItem *lhs_item = lhs.index_array_.at(i); + const FileIndexItem *rhs_item = rhs.index_array_.at(i); + if (*lhs_item == *rhs_item) { + } else { + bret = false; + } + } + } + + return bret; +} + +TEST(test_ob_log_miner_file_index, FileIndexItem) +{ + FileIndexItem item, item1; + const int64_t buf_len = 300; + char buf[buf_len]; + int64_t pos = 0; + const char *buf1 = "1:2:3\n"; + EXPECT_EQ(OB_SUCCESS, item.deserialize(buf1, strlen(buf1), pos)); + EXPECT_EQ(item.file_id_, 1); + EXPECT_EQ(item.range_.min_commit_ts_, 2); + EXPECT_EQ(item.range_.max_commit_ts_, 3); + pos = 0; + const char *buf2 = "1:2:3"; + EXPECT_EQ(OB_SIZE_OVERFLOW, item.deserialize(buf2, strlen(buf2), pos)); + pos = 0; + const char *buf3 = "1:a:3\n"; + EXPECT_EQ(OB_INVALID_DATA, item.deserialize(buf3, strlen(buf3), pos)); + pos = 0; + const char *buf4 = "1|2|3\n"; + EXPECT_EQ(OB_INVALID_DATA, item.deserialize(buf4, strlen(buf4), pos)); + pos = 0; + + for (int i = 0; i < 10000; i++) { + item1.file_id_ = rand(); + item1.range_.min_commit_ts_ = rand(); + item1.range_.max_commit_ts_ = rand(); + EXPECT_EQ(OB_SUCCESS, item1.serialize(buf, buf_len, pos)); + EXPECT_EQ(item1.get_serialize_size(), pos); + pos = 0; + EXPECT_EQ(OB_SUCCESS, item.deserialize(buf, buf_len, pos)); + EXPECT_EQ(item, item1); + pos = 0; + } +} + +TEST(test_ob_log_miner_file_index, FileIndex) +{ + FileIndex index2, index1,index; + FileIndexItem *item = nullptr; + const int64_t buf_size = 1000; + char buf[buf_size]; + char buffer[buf_size]; + int64_t pos = 0; + int64_t position = 0; + const char *buf1 = "0:2:3\n1:3:3\n2:4:7\n"; + EXPECT_EQ(OB_SUCCESS, index.deserialize(buf1, strlen(buf1), pos)); + EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(0,2,3)); + EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(1,3,3)); + EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(2,4,7)); + EXPECT_EQ(index, index1); + EXPECT_EQ(OB_ERROR_OUT_OF_RANGE, index.get_index_item(-1, item)); + EXPECT_EQ(OB_SUCCESS, index.get_index_item(0, item)); + EXPECT_EQ(OB_SUCCESS, index.get_index_item(1, item)); + EXPECT_EQ(OB_SUCCESS, index.get_index_item(2, item)); + EXPECT_EQ(OB_ERROR_OUT_OF_RANGE, index.get_index_item(3, item)); + pos = 0; + index.reset(); + index1.reset(); + index2.reset(); + const char *buf2 = "1:2:3\n4:5:6\n3:4:5\n"; + EXPECT_EQ(OB_SUCCESS, index.deserialize(buf2, strlen(buf2), pos)); + EXPECT_EQ(OB_ERR_UNEXPECTED, index.get_index_item(0, item)); + pos = 0; + index.reset(); + index1.reset(); + index2.reset(); + const char *buf3 = "0:1:2\n1:2:3\n"; + EXPECT_EQ(OB_SUCCESS, index.deserialize(buf3, strlen(buf3), pos)); + EXPECT_EQ(strlen(buf3), index.get_index_file_len()); + pos = 0; + index.reset(); + index1.reset(); + index2.reset(); + for(int i = 0; i < 1000; i++) { + index.reset(); + index1.reset(); + int item_cnt = abs(rand()) % 10 + 1; + for (int j = 0; j < item_cnt; j++) { + FileIndexItem item; + item.reset(j, rand(), rand()); + EXPECT_EQ(OB_SUCCESS, index1.insert_index_item(item)); + EXPECT_EQ(OB_SUCCESS, item.serialize(buffer, buf_size, position)); + } + EXPECT_EQ(position, index1.get_serialize_size()); + EXPECT_EQ(position, index1.get_index_file_len()); + int64_t data_len = position; + position = 0; + EXPECT_EQ(OB_SUCCESS, index2.deserialize(buffer, data_len, position)); + EXPECT_EQ(index1, index2); + EXPECT_EQ(OB_SUCCESS, index1.serialize(buf, buf_size, pos)); + data_len = pos; + pos = 0; + EXPECT_EQ(OB_SUCCESS, index.deserialize(buf, data_len, pos)); + EXPECT_EQ(index, index1); + pos = 0; + position = 0; + index.reset(); + index1.reset(); + index2.reset(); + } +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_file_index.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_file_index.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_file_manager.cpp b/unittest/logminer/test_ob_log_miner_file_manager.cpp new file mode 100644 index 0000000000..8847d8c7db --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_file_manager.cpp @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define private public +#include "ob_log_miner_file_manager.h" +#undef private +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_ob_log_miner_file_manager, GenFileHeader) +{ + ObLogMinerFileManager manager; + const int64_t path_size = 100; + char cwd_buf[path_size]; + ObArenaAllocator alloc; + char *header = nullptr; + int64_t data_len = 0; + EXPECT_NE(nullptr, getcwd(cwd_buf, path_size)); + char path[OB_MAX_URI_LENGTH]; + char rm_dir_command[OB_MAX_URI_LENGTH]; + EXPECT_EQ(OB_SUCCESS, databuff_printf(path, sizeof(path), "file://%s/logminer_output", cwd_buf)); + EXPECT_EQ(OB_SUCCESS, databuff_printf(rm_dir_command, sizeof(rm_dir_command), "rm -rf %s/logminer_output", cwd_buf)); + EXPECT_EQ(OB_SUCCESS, + manager.init(path, RecordFileFormat::CSV, ObLogMinerFileManager::FileMgrMode::ANALYZE)); + EXPECT_EQ(OB_SUCCESS, manager.generate_data_file_header_(alloc, header, data_len)); + EXPECT_STREQ(header, "TENANT_ID,TRANS_ID,PRIMARY_KEY,TENANT_NAME,DATABASE_NAME," + "TABLE_NAME,OPERATION,OPERATION_CODE,COMMIT_SCN,COMMIT_TIMESTAMP,SQL_REDO,SQL_UNDO,ORG_CLUSTER_ID\n"); + + system(rm_dir_command); +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_file_manager.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_file_manager.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/unittest/logminer/test_ob_log_miner_file_meta.cpp b/unittest/logminer/test_ob_log_miner_file_meta.cpp new file mode 100644 index 0000000000..b4470ec718 --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_file_meta.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_miner_file_meta.h" +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_log_miner_file_meta, SerializeFunc) +{ + ObLogMinerFileMeta file_meta, file_meta1; + const int64_t buf_len = 300; + char buf[buf_len]; + int64_t pos = 0; + const char *buf1 = "MIN_COMMIT_TS=1\nMAX_COMMIT_TS=2\nDATA_LEN=3\n"; + EXPECT_EQ(OB_SUCCESS, file_meta.deserialize(buf1, strlen(buf1), pos)); + EXPECT_EQ(pos, strlen(buf1)); + EXPECT_EQ(file_meta.range_.min_commit_ts_, 1); + EXPECT_EQ(file_meta.range_.max_commit_ts_, 2); + EXPECT_EQ(file_meta.data_length_, 3); + pos = 0; + const char *buf2 = "MIN_COMMIT_TS=1a\nMAX_COMMIT_TS=2\nDATA_LEN=3\n"; + EXPECT_EQ(OB_INVALID_DATA, file_meta.deserialize(buf2, strlen(buf2), pos)); + pos = 0; + const char *buf3 = "MIN_COMMIT_TS=11\nMAX_COMMIT_TS=2\nDATA_LEN=3"; + EXPECT_EQ(OB_SIZE_OVERFLOW, file_meta.deserialize(buf3, strlen(buf3), pos)); + pos = 0; + + for (int i = 0; i < 10000; i++) { + file_meta1.data_length_ = rand(); + file_meta1.range_.min_commit_ts_ = rand(); + file_meta1.range_.max_commit_ts_ = rand(); + EXPECT_EQ(OB_SUCCESS, file_meta1.serialize(buf, buf_len, pos)); + EXPECT_EQ(pos, file_meta1.get_serialize_size()); + pos = 0; + EXPECT_EQ(OB_SUCCESS, file_meta.deserialize(buf, buf_len, pos)); + EXPECT_EQ(file_meta, file_meta1); + pos = 0; + } +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_file_meta.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_file_meta.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_progress_range.cpp b/unittest/logminer/test_ob_log_miner_progress_range.cpp new file mode 100644 index 0000000000..8986733819 --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_progress_range.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_miner_progress_range.h" +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_log_miner_progress_range, SerializeFunc) +{ + ObLogMinerProgressRange range, range1; + const int64_t buf_len = 100; + char buf[buf_len]; + int64_t pos = 0; + const char *buf1 = "MIN_COMMIT_TS=1\nMAX_COMMIT_TS=2\n"; + EXPECT_EQ(OB_SUCCESS, range.deserialize(buf1, strlen(buf1), pos)); + EXPECT_EQ(range.min_commit_ts_, 1); + EXPECT_EQ(range.max_commit_ts_, 2); + pos = 0; + const char *buf2 = "MIN_COMMIT_TS=aaa\nMAX_COMMIT_TS=2\n"; + EXPECT_EQ(OB_INVALID_DATA, range.deserialize(buf2, strlen(buf2), pos)); + pos = 0; + const char *buf3 = "MIN_COMMIT_TS:1\nMAX_COMMIT_TS:2\n"; + EXPECT_EQ(OB_INVALID_DATA, range.deserialize(buf3, strlen(buf3), pos)); + pos = 0; + const char *buf4 = "MIN_COMMIT_TS=1MAX_COMMIT_TS=2\n"; + EXPECT_EQ(OB_INVALID_DATA, range.deserialize(buf4, strlen(buf4), pos)); + pos = 0; + for (int i = 0; i < 10000; i++) { + range1.min_commit_ts_ = rand(); + range1.max_commit_ts_ = rand(); + EXPECT_EQ(range1.serialize(buf, buf_len, pos), OB_SUCCESS); + EXPECT_EQ(pos, range1.get_serialize_size()); + pos = 0; + EXPECT_EQ(range.deserialize(buf, buf_len, pos), OB_SUCCESS); + EXPECT_EQ(range.get_serialize_size(), pos); + EXPECT_EQ(range, range1); + pos = 0; + } +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_progress_range.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_progress_range.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_record.cpp b/unittest/logminer/test_ob_log_miner_record.cpp new file mode 100644 index 0000000000..e7b0c24b98 --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_record.cpp @@ -0,0 +1,795 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/oblog/ob_log.h" +#include "ob_log_miner_test_utils.h" +#include "gtest/gtest.h" +#define private public +#include "ob_log_miner_record.h" +#undef private +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_ob_log_miner_record, InitObLogMinerRecord) +{ + ObArenaAllocator allocator("TestLogMnrRec"); + ObLogMinerBR *br = nullptr; + ObLogMinerRecord record; + const int buf_cnt = 10; + binlogBuf *new_buf = static_cast(allocator.alloc(sizeof(binlogBuf) * buf_cnt)); + binlogBuf *old_buf = static_cast(allocator.alloc(sizeof(binlogBuf) * buf_cnt)); + for (int i = 0; i < 10; i++) { + new_buf[i].buf = static_cast(allocator.alloc(1024)); + new_buf[i].buf_size = 1024; + new_buf[i].buf_used_size = 0; + old_buf[i].buf = static_cast(allocator.alloc(1024)); + old_buf[i].buf_size = 1024; + old_buf[i].buf_used_size = 0; + } + record.set_allocator(&allocator); + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "t1.db1", "tbl1", 4, "id", "1", nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("t1", record.tenant_name_.ptr()); + EXPECT_STREQ("db1", record.database_name_.ptr()); + EXPECT_STREQ("tbl1", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db1`.`tbl1` (`id`) VALUES ('1');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db1`.`tbl1` WHERE `id`='1' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", 8, "id", "1", "2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "name", "aaa", "bbb", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `id`='1', `name`='aaa' WHERE `id`='2' AND `name`='bbb' LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `id`='2', `name`='bbb' WHERE `id`='1' AND `name`='aaa' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", 8, "id", nullptr , "2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "name", nullptr, "bbb", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='bbb' LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', 'bbb');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDDL, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "", 4, "ddl_stmt_str", "CREATE TABLE T1(ID INT PRIMARY KEY);" , nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("CREATE TABLE T1(ID INT PRIMARY KEY);", record.redo_stmt_.ptr()); + EXPECT_STREQ("/* NO SQL_UNDO GENERATED */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "gbk", 8, + "id", nullptr , "2",static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "name", nullptr, "\xB0\xC2\xD0\xC7\xB1\xB4\xCB\xB9", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF' LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', '\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf16", 4, "name", nullptr, "\x59\x65\x66\x1F\x8D\x1D\x65\xAF", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `name`='\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF' LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`) VALUES ('\xE5\xA5\xA5\xE6\x98\x9F\xE8\xB4\x9D\xE6\x96\xAF');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "latin1", 8, "id", nullptr , "2", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "name", nullptr, "\x63\x69\xe0\x6f", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='\x63\x69\xc3\xa0\x6f' LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', '\x63\x69\xc3\xa0\x6f');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "gb18030", 8, + "id", nullptr , "2",static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING), + "name", nullptr, "\x95\xcb\x95\xcb\xc6\xe4\xd2\xf5\xa3\xac\xc6\xe4\xec\x59\xf2\xb3\xf2\xb3", + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `id`='2' AND `name`='\xe6\x9b\x80\xe6\x9b\x80\xe5\x85\xb6\xe9\x98\xb4\xef\xbc\x8c\xe5\x85\xb6\xe9\x9d\x81\xe8\x99\xba\xe8\x99\xba' LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`id`, `name`) VALUES ('2', '\xe6\x9b\x80\xe6\x9b\x80\xe5\x85\xb6\xe9\x98\xb4\xef\xbc\x8c\xe5\x85\xb6\xe9\x9d\x81\xe8\x99\xba\xe8\x99\xba');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "gb18030", 4, + "id", nullptr , "2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"id\"=2 and ROWNUM=1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES (2);", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "gb18030", 4, + "id", nullptr , "2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"id\"='2' and ROWNUM=1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES ('2');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "gb18030", 4, + "id", "2" , nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_VAR_STRING)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES ('2');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"id\"='2' and ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "gb18030", 4, + "id", "2" , nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_BLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"id\") VALUES ('2');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE 0=DBMS_LOB.COMPARE(\"id\", '2') and ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 4, + "name", "ABCD" , nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_BLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"name\") VALUES (HEXTORAW('41424344'));", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE 0=DBMS_LOB.COMPARE(\"name\", HEXTORAW('41424344')) and ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "binary", 4, + "name", "ABCD" , nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_BLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`) VALUES (UNHEX('41424344'));", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `name`=UNHEX('41424344') LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, "name", "ABCD" , nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING), + "col2", "XXX", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING)); + br->get_br()->getTableMeta()->setUKs("name"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`) VALUES ('ABCD', 'XXX');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `name`='ABCD' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, "name", "ABCD" , nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING), + "col2", "XXX", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING)); + br->get_br()->getTableMeta()->setPKs("col2"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`) VALUES ('ABCD', 'XXX');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`='XXX' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, "name", "ABCD" , nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING), + "col2", "XXX", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING), + "col3", "val3", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING)); + br->get_br()->getTableMeta()->setPKs("col2,col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`, `col3`) VALUES ('ABCD', 'XXX', 'val3');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`='XXX' AND `col3`='val3' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, "name", "ABCD" , nullptr, + static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING), + "col2", "XXX", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING), + "col3", "val3", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_STRING)); + br->get_br()->getTableMeta()->setUKs("col2,col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`name`, `col2`, `col3`) VALUES ('ABCD', 'XXX', 'val3');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`='XXX' AND `col3`='val3' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG), + "col2", "2", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG), + "col3", "3", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col2,col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES (1, 2, 3);", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col2`=2 AND `col3`=3 LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_BIT), + "col2", "1836032", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_BIT)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES (b'1', b'111000000010000000000');", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1`=b'1' AND `col2`=b'111000000010000000000' LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); +} + +TEST(test_ob_log_miner_record, LobTypeInMySqlMode) +{ + ObArenaAllocator allocator("TestLogMnrRec"); + ObLogMinerBR *br = nullptr; + ObLogMinerRecord record; + const int buf_cnt = 10; + binlogBuf *new_buf = static_cast(allocator.alloc(sizeof(binlogBuf) * buf_cnt)); + binlogBuf *old_buf = static_cast(allocator.alloc(sizeof(binlogBuf) * buf_cnt)); + for (int i = 0; i < 10; i++) { + new_buf[i].buf = static_cast(allocator.alloc(1024)); + new_buf[i].buf_size = 1024; + new_buf[i].buf_used_size = 0; + old_buf[i].buf = static_cast(allocator.alloc(1024)); + old_buf[i].buf_size = 1024; + old_buf[i].buf_used_size = 0; + } + record.set_allocator(&allocator); + + + // mysql multimode + // insert without key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", "{\"key\": \"value\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES ('{\"key\": \"value\"}', ST_GeomFromText('POINT(0 1)'));",record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1`=cast('{\"key\": \"value\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // insert with key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", "{\"key\": \"value\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES ('{\"key\": \"value\"}', ST_GeomFromText('POINT(0 1)'), 1);", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=1 LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value insert without key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES (NULL, ST_GeomFromText('POINT(0 1)'));",record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1` IS NULL AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value insert with key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", "{\"key\": \"value\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES ('{\"key\": \"value\"}', NULL, 1);", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=1 LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // update without key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)') WHERE `col1`=cast('{\"key\": \"old\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"old\"}', `col2`=ST_GeomFromText('POINT(2 3)') WHERE `col1`=cast('{\"key\": \"new\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // update with key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "1", "2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)'), `col3`=1 WHERE `col3`=2 LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"old\"}', `col2`=ST_GeomFromText('POINT(2 3)'), `col3`=2 WHERE `col3`=1 LIMIT 1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value update without key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", "{\"key\": \"new\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)') WHERE `col1` IS NULL AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`=NULL, `col2`=ST_GeomFromText('POINT(2 3)') WHERE `col1`=cast('{\"key\": \"new\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(0 1)')) LIMIT 1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value update with key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", "{\"key\": \"new\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "POINT(0 1)", "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`='{\"key\": \"new\"}', `col2`=ST_GeomFromText('POINT(0 1)'), `col3`=1 WHERE `col3` IS NULL LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE `db2`.`tbl2` SET `col1`=NULL, `col2`=ST_GeomFromText('POINT(2 3)'), `col3`=NULL WHERE `col3`=1 LIMIT 1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // delete without key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", nullptr, "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1`=cast('{\"key\": \"old\"}'as json) AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES ('{\"key\": \"old\"}', ST_GeomFromText('POINT(2 3)'));", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // delete with key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", nullptr, "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, "2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=2 LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES ('{\"key\": \"old\"}', ST_GeomFromText('POINT(2 3)'), 2);", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value delete without key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 8, + "col1", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col1` IS NULL AND ST_Equals(`col2`, ST_GeomFromText('POINT(2 3)')) LIMIT 1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`) VALUES (NULL, ST_GeomFromText('POINT(2 3)'));/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value delete with key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::MYSQL, + "tenant2.db2", "tbl2", "utf8mb4", 12, + "col1", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "POINT(2 3)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, "1", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col3"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM `db2`.`tbl2` WHERE `col3`=1 LIMIT 1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO `db2`.`tbl2` (`col1`, `col2`, `col3`) VALUES (NULL, ST_GeomFromText('POINT(2 3)'), 1);/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); +} + +TEST(test_ob_log_miner_record, LobTypeInOracleMode) +{ + ObArenaAllocator allocator("TestLogMnrRec"); + ObLogMinerBR *br = nullptr; + ObLogMinerRecord record; + const int buf_cnt = 10; + binlogBuf *new_buf = static_cast(allocator.alloc(sizeof(binlogBuf) * buf_cnt)); + binlogBuf *old_buf = static_cast(allocator.alloc(sizeof(binlogBuf) * buf_cnt)); + for (int i = 0; i < 10; i++) { + new_buf[i].buf = static_cast(allocator.alloc(1024)); + new_buf[i].buf_size = 1024; + new_buf[i].buf_used_size = 0; + old_buf[i].buf = static_cast(allocator.alloc(1024)); + old_buf[i].buf_size = 1024; + old_buf[i].buf_used_size = 0; + } + record.set_allocator(&allocator); + // multimode + // insert without key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 16, + "col1", "{\"key\": \"value\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), 'abc', 'AABB1122');",record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE JSON_EQUAL(\"col1\", '{\"key\": \"value\"}') AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='abc' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // insert with key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 20, + "col1", "{\"key\": \"value\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB), + "col5", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col5"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), 'abc', 'AABB1122', 1);", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=1 and ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value insert without key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 16, + "col1", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES (NULL, SDO_GEOMETRY('POINT(0 1)', NULL), 'abc', 'AABB1122');",record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col1\" IS NULL AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='abc' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value insert with key + br = build_logminer_br(new_buf, old_buf, EINSERT, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 20, + "col1", "{\"key\": \"value\"}", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB), + "col5", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col5"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', NULL, 'abc', 'AABB1122', 1);", record.redo_stmt_.ptr()); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=1 and ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // update without key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 16, + "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "SRID=NULL;POINT(0 1)", "SRID=NULL;POINT(2 1)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", "cba", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", "1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL)," + " \"col3\"='abc', \"col4\"='AABB1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"old\"}') AND " + "\"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL) AND \"col3\"='cba' AND 0=DBMS_LOB.COMPARE(\"col4\", '1122')" + " AND ROWNUM=1;/* POTENTIALLY INACCURATE */", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL)," + " \"col3\"='cba', \"col4\"='1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"new\"}') AND " + "\"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='abc' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122')" + " AND ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // update with key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 20, + "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "SRID=NULL;POINT(0 1)", "SRID=NULL;POINT(2 1)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", "cba", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", "1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB), + "col5", "1", "2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col5"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL), \"col3\"='abc', \"col4\"='AABB1122', \"col5\"=1 WHERE \"col5\"=2 AND ROWNUM=1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL), \"col3\"='cba', \"col4\"='1122', \"col5\"=2 WHERE \"col5\"=1 AND ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value update without key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 16, + "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "SRID=NULL;POINT(0 1)", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, nullptr, drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", "1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL)," + " \"col3\"=NULL, \"col4\"='AABB1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"old\"}') AND \"col2\" IS NULL AND " + "\"col3\" IS NULL AND 0=DBMS_LOB.COMPARE(\"col4\", '1122') AND ROWNUM=1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=NULL, \"col3\"=NULL," + " \"col4\"='1122' WHERE JSON_EQUAL(\"col1\", '{\"key\": \"new\"}') AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL)" + " AND \"col3\" IS NULL AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') AND ROWNUM=1;/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value update with key + br = build_logminer_br(new_buf, old_buf, EUPDATE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 20, + "col1", "{\"key\": \"new\"}", "{\"key\": \"old\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", "SRID=NULL;POINT(0 1)", "SRID=NULL;POINT(2 1)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", "abc", "cba", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", "AABB1122", "1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB), + "col5", "1", nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col5"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"new\"}', \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL), \"col3\"='abc', \"col4\"='AABB1122', \"col5\"=1 WHERE \"col5\" IS NULL AND ROWNUM=1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("UPDATE \"db2\".\"tbl2\" SET \"col1\"='{\"key\": \"old\"}', \"col2\"=SDO_GEOMETRY('POINT(2 1)', NULL), \"col3\"='cba', \"col4\"='1122', \"col5\"=NULL WHERE \"col5\"=1 AND ROWNUM=1;", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // delete without key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 16, + "col1", nullptr, "{\"key\": \"value\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "SRID=NULL;POINT(0 1)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, "abc", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", nullptr, "AABB1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE JSON_EQUAL(\"col1\", '{\"key\": \"value\"}') AND \"col2\"=SDO_GEOMETRY('POINT(0 1)', NULL) AND \"col3\"='abc' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), 'abc', 'AABB1122');", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // delete with key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 20, + "col1", nullptr, "{\"key\": \"value\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "SRID=NULL;POINT(0 1)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, "abc", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", nullptr, "AABB1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB), + "col5", nullptr, "2", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col5"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=2 and ROWNUM=1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), 'abc', 'AABB1122', 2);", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value delete without key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 16, + "col1", nullptr, "{\"key\": \"value\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, nullptr, static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, "abc", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", nullptr, "AABB1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB)); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE JSON_EQUAL(\"col1\", '{\"key\": \"value\"}') AND \"col2\" IS NULL AND \"col3\"='abc' AND 0=DBMS_LOB.COMPARE(\"col4\", 'AABB1122') and ROWNUM=1;/* POTENTIALLY INACCURATE */",record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES ('{\"key\": \"value\"}', NULL, 'abc', 'AABB1122');/* POTENTIALLY INACCURATE */", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); + + // null value delete with key + br = build_logminer_br(new_buf, old_buf, EDELETE, lib::Worker::CompatMode::ORACLE, + "tenant2.db2", "tbl2", "utf8mb4", 20, + "col1", nullptr, "{\"key\": \"value\"}", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_JSON), + "col2", nullptr, "SRID=NULL;POINT(0 1)", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_GEOMETRY), + "col3", nullptr, "abc", drcmsg_field_types::DRCMSG_TYPE_ORA_XML, + "col4", nullptr, "AABB1122", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_ORA_CLOB), + "col5", nullptr, "1", static_cast(obmysql::EMySQLFieldType::MYSQL_TYPE_LONG)); + br->get_br()->getTableMeta()->setUKs("col5"); + EXPECT_EQ(OB_SUCCESS, record.init(*br)); + EXPECT_STREQ("tenant2", record.tenant_name_.ptr()); + EXPECT_STREQ("db2", record.database_name_.ptr()); + EXPECT_STREQ("tbl2", record.table_name_.ptr()); + EXPECT_EQ(OB_SUCCESS, record.build_stmts(*br)); + EXPECT_STREQ("DELETE FROM \"db2\".\"tbl2\" WHERE \"col5\"=1 and ROWNUM=1;", record.redo_stmt_.ptr()); + EXPECT_STREQ("INSERT INTO \"db2\".\"tbl2\" (\"col1\", \"col2\", \"col3\", \"col4\", \"col5\") VALUES ('{\"key\": \"value\"}', SDO_GEOMETRY('POINT(0 1)', NULL), 'abc', 'AABB1122', 1);", record.undo_stmt_.ptr()); + destroy_miner_br(br); + record.destroy(); +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_record.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_record.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_record_converter.cpp b/unittest/logminer/test_ob_log_miner_record_converter.cpp new file mode 100644 index 0000000000..f2b73d4d5f --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_record_converter.cpp @@ -0,0 +1,313 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_log_miner_timezone_getter.h" +#include "ob_log_miner_test_utils.h" +#include "gtest/gtest.h" + +#define private public +#include "ob_log_miner_record_converter.h" +#undef private + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_ob_log_miner_record_converter, CsvConverterWriteType) +{ + ObLogMinerRecordCsvConverter converter; + ObConcurrentFIFOAllocator alloc; + ObStringBuffer str_buf(&alloc); + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + EXPECT_EQ(OB_SUCCESS, converter.write_csv_string_escape_("'aaaa\"\"bbbbb'", str_buf)); + EXPECT_STREQ("\"'aaaa\"\"\"\"bbbbb'\"", str_buf.ptr()); + str_buf.reset(); +} + +TEST(test_ob_log_miner_record_converter, JsonConverterWriteType) +{ + ObLogMinerRecordJsonConverter converter; + ObConcurrentFIFOAllocator alloc; + ObStringBuffer str_buf(&alloc); + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + EXPECT_EQ(OB_SUCCESS, converter.write_json_key_("aaa", str_buf)); + EXPECT_STREQ("\"aaa\":", str_buf.ptr()); + str_buf.reset(); + EXPECT_EQ(OB_SUCCESS, converter.write_json_string_escape_("'aaaa\"\"bbbbb'", str_buf)); + EXPECT_STREQ("\"'aaaa\\\"\\\"bbbbb'\"", str_buf.ptr()); + str_buf.reset(); + EXPECT_EQ(OB_SUCCESS, converter.write_json_string_escape_("\n\b\t", str_buf)); + EXPECT_STREQ("\"\\n\\b\\t\"", str_buf.ptr()); + str_buf.reset(); +} + +TEST(test_ob_log_miner_record_converter, CsvConverterWriteRecord) +{ + ObLogMinerRecordCsvConverter converter; + ObConcurrentFIFOAllocator alloc; + EXPECT_EQ(OB_SUCCESS, LOGMINER_TZ.set_timezone("+8:00")); + bool is_written = false; + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + ObStringBuffer str_buf(&alloc); + ObLogMinerRecord *rec = nullptr; + const char *pkarr1[] = {"aaa", "bbb"}; + const char *ukarr1[] = {"ccc"}; + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222, + "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');", + "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';"); + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "1002,345,\"aaa/bbb\",\"test_tenant\",\"test\",\"sbtest1\",\"INSERT\",1,1645539742222222000,\"2022-02-22 22:22:22.222222\"," + "\"INSERT INTO \"\"test\"\".\"\"sbtest1\"\"(\"\"aaa\"\",\"\"bbb\"\",\"\"ccc\"\") VALUES('1','2','3');\"," + "\"DELETE FROM \"\"test\"\".\"\"sbtest1\"\" WHERE \"\"aaa\"\"='1' and \"\"bbb\"\"='2' and \"\"ccc\"\"='3';\"," + "1\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EUPDATE, 0, + "UPDATE \"test\".\"sbtest1\" SET \"aaa\" = '1', \"bbb\" = '2', \"ccc\" = '3' WHERE " + "\"aaa\" = '4' AND \"bbb\" = '5' and \"ccc\" = '6' LIMIT 1;", + "UPDATE \"test\".\"sbtest1\" SET \"aaa\" = '4', \"bbb\" = '5', \"ccc\" = '6' WHERE " + "\"aaa\" = '1' AND \"bbb\" = '2' and \"ccc\" = '3' LIMIT 1;"); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "1002,345,\"aaa/bbb\",\"test_tenant\",\"test\",\"sbtest1\",\"UPDATE\",2,0,\"1970-01-01 08:00:00.000000\"," + "\"UPDATE \"\"test\"\".\"\"sbtest1\"\" SET \"\"aaa\"\" = '1', \"\"bbb\"\" = '2', \"\"ccc\"\" = '3' WHERE " + "\"\"aaa\"\" = '4' AND \"\"bbb\"\" = '5' and \"\"ccc\"\" = '6' LIMIT 1;\"," + "\"UPDATE \"\"test\"\".\"\"sbtest1\"\" SET \"\"aaa\"\" = '4', \"\"bbb\"\" = '5', \"\"ccc\"\" = '6' WHERE " + "\"\"aaa\"\" = '1' AND \"\"bbb\"\" = '2' and \"\"ccc\"\" = '3' LIMIT 1;\"," + "1\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "", 345, nullptr,0, + nullptr, 0, nullptr, EDDL, 4611686018427387, + "CREATE TABLE t1(id INT, name TEXT);", + nullptr); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "1002,345,\"\",\"test_tenant\",\"test\",\"\",\"DDL\",4,4611686018427387000,\"2116-02-21 07:53:38.427387\"," + "\"CREATE TABLE t1(id INT, name TEXT);\"," + "\"\"," + "1\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EDELETE, 0, + "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';", + "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');" + ); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "1002,345,\"aaa/bbb\",\"test_tenant\",\"test\",\"sbtest1\",\"DELETE\",3,0,\"1970-01-01 08:00:00.000000\"," + "\"DELETE FROM \"\"test\"\".\"\"sbtest1\"\" WHERE \"\"aaa\"\"='1' and \"\"bbb\"\"='2' and \"\"ccc\"\"='3';\"," + "\"INSERT INTO \"\"test\"\".\"\"sbtest1\"\"(\"\"aaa\"\",\"\"bbb\"\",\"\"ccc\"\") VALUES('1','2','3');\"," + "1\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); +} + +TEST(test_ob_log_miner_record_converter, RedoSqlConverterWriteRecord) +{ + ObLogMinerRecordRedoSqlConverter converter; + ObConcurrentFIFOAllocator alloc; + bool is_written = false; + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + ObStringBuffer str_buf(&alloc); + ObLogMinerRecord *rec = nullptr; + const char *pkarr1[] = {"aaa", "bbb"}; + const char *ukarr1[] = {"ccc"}; + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222, + "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');", + "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';"); + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "", 345, nullptr,0, + nullptr, 0, nullptr, EDDL, 4611686018427387, + "CREATE TABLE t1(id INT, name TEXT);", + nullptr); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "CREATE TABLE t1(id INT, name TEXT);\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "", 345, nullptr,0, + nullptr, 0, nullptr, EBEGIN, 4611686018427387, + nullptr, + nullptr); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(false, is_written); + destroy_miner_record(rec); +} + +TEST(test_ob_log_miner_record_converter, UndoSqlConverterWriteRecord) +{ + ObLogMinerRecordUndoSqlConverter converter; + ObConcurrentFIFOAllocator alloc; + bool is_written = false; + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + ObStringBuffer str_buf(&alloc); + ObLogMinerRecord *rec = nullptr; + const char *pkarr1[] = {"aaa", "bbb"}; + const char *ukarr1[] = {"ccc"}; + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222, + "INSERT INTO \"test\".\"sbtest1\"(\"aaa\",\"bbb\",\"ccc\") VALUES('1','2','3');", + "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';"); + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "DELETE FROM \"test\".\"sbtest1\" WHERE \"aaa\"='1' and \"bbb\"='2' and \"ccc\"='3';\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "", 345, nullptr, 0, + nullptr, 0, nullptr, EDDL, 4611686018427387, + "CREATE TABLE t1(id INT, name TEXT);", + nullptr); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(false, is_written); + str_buf.reset(); + destroy_miner_record(rec); + + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "", 345, nullptr,0, + nullptr, 0, nullptr, EBEGIN, 4611686018427387, + nullptr, + nullptr); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(false, is_written); + destroy_miner_record(rec); +} + +TEST(test_ob_log_miner_record_converter, JsonConverterWriteRecord) +{ + ObLogMinerRecordJsonConverter converter; + ObConcurrentFIFOAllocator alloc; + EXPECT_EQ(OB_SUCCESS, LOGMINER_TZ.set_timezone("+8:00")); + bool is_written = false; + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + ObStringBuffer str_buf(&alloc); + ObLogMinerRecord *rec = nullptr; + const char *pkarr1[] = {"aaa", "bbb"}; + const char *ukarr1[] = {"ccc"}; + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EINSERT, 1645539742222222, + "INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');", + "DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;"); + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"aaa/bbb\",\"TENANT_NAME\":\"test_tenant\",\"DATABASE_NAME\":\"test\"," + "\"TABLE_NAME\":\"sbtest1\",\"OPERATION\":\"INSERT\",\"OPERATION_CODE\":1,\"COMMIT_SCN\":1645539742222222000," + "\"COMMIT_TIMESTAMP\":\"2022-02-22 22:22:22.222222\"," + "\"SQL_REDO\":\"INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');\"," + "\"SQL_UNDO\":\"DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;\"," + "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest2", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EUPDATE, 0, + "UPDATE `test`.`sbtest2` SET `aaa`='44', `bbb`='55', `ccc`='66' WHERE `aaa`='11' AND `bbb`='22' AND `ccc`='33' LIMIT 1;", + "UPDATE `test`.`sbtest2` SET `aaa`='11', `bbb`='22', `ccc`='33' WHERE `aaa`='44' AND `bbb`='55' AND `ccc`='66' LIMIT 1;"); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"aaa/bbb\",\"TENANT_NAME\":\"test_tenant\"," + "\"DATABASE_NAME\":\"test\",\"TABLE_NAME\":\"sbtest2\",\"OPERATION\":\"UPDATE\",\"OPERATION_CODE\":2," + "\"COMMIT_SCN\":0,\"COMMIT_TIMESTAMP\":\"1970-01-01 08:00:00.000000\"," + "\"SQL_REDO\":\"UPDATE `test`.`sbtest2` SET `aaa`='44', `bbb`='55', `ccc`='66' WHERE `aaa`='11' AND `bbb`='22' AND `ccc`='33' LIMIT 1;\"," + "\"SQL_UNDO\":\"UPDATE `test`.`sbtest2` SET `aaa`='11', `bbb`='22', `ccc`='33' WHERE `aaa`='44' AND `bbb`='55' AND `ccc`='66' LIMIT 1;\"," + "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "", 345, nullptr,0, + nullptr, 0, nullptr, EDDL, 4611686018427387, + "CREATE TABLE `sbtest2` (\n `aaa` varchar(100) NOT NULL,\n `bbb` varchar(100) NOT NULL,\n `ccc` varchar(100) DEFAULT NULL\n);", + nullptr); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"\",\"TENANT_NAME\":\"test_tenant\",\"DATABASE_NAME\":\"test\"," + "\"TABLE_NAME\":\"\",\"OPERATION\":\"DDL\",\"OPERATION_CODE\":4,\"COMMIT_SCN\":4611686018427387000,\"COMMIT_TIMESTAMP\":\"2116-02-21 07:53:38.427387\"," + "\"SQL_REDO\":\"CREATE TABLE `sbtest2` (\\n `aaa` varchar(100) NOT NULL,\\n `bbb` varchar(100) NOT NULL,\\n `ccc` varchar(100) DEFAULT NULL\\n);\",\"SQL_UNDO\":\"\"," + "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); + rec = build_logminer_record(alloc, lib::Worker::CompatMode::MYSQL, + 1002, 1, "test_tenant", "test", "sbtest1", 345, pkarr1, sizeof(pkarr1)/ sizeof(const char*), + ukarr1, sizeof(ukarr1)/sizeof(const char*), "a/b/c/d/e", EDELETE, 0, + "DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;", + "INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');" + ); + is_written = false; + EXPECT_EQ(OB_SUCCESS, converter.write_record(*rec, str_buf, is_written)); + EXPECT_EQ(true, is_written); + EXPECT_STREQ( + "{\"TENANT_ID\":1002,\"TRANS_ID\":345,\"PRIMARY_KEY\":\"aaa/bbb\",\"TENANT_NAME\":\"test_tenant\",\"DATABASE_NAME\":\"test\"," + "\"TABLE_NAME\":\"sbtest1\",\"OPERATION\":\"DELETE\",\"OPERATION_CODE\":3,\"COMMIT_SCN\":0," + "\"COMMIT_TIMESTAMP\":\"1970-01-01 08:00:00.000000\"," + "\"SQL_REDO\":\"DELETE FROM `test`.`sbtest1` WHERE `aaa`='1' AND `bbb`='2' AND `ccc`='3' LIMIT 1;\"," + "\"SQL_UNDO\":\"INSERT INTO `test`.`sbtest1` (`aaa`, `bbb`, `ccc`) VALUES ('1', '2', '3');\"," + "\"ORG_CLUSTER_ID\":1}\n", str_buf.ptr()); + str_buf.reset(); + destroy_miner_record(rec); +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_record_converter.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_record_converter.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/logminer/test_ob_log_miner_utils.cpp b/unittest/logminer/test_ob_log_miner_utils.cpp new file mode 100644 index 0000000000..e0bd0e0cae --- /dev/null +++ b/unittest/logminer/test_ob_log_miner_utils.cpp @@ -0,0 +1,310 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/allocator/ob_concurrent_fifo_allocator.h" //ObConcurrentFIFOAllocator +#include "lib/allocator/page_arena.h" +#include "ob_log_miner_utils.h" +#include "gtest/gtest.h" + +namespace oceanbase +{ +namespace oblogminer +{ + +TEST(test_ob_log_miner_utils, ObLogMinerDeepCopyCstring) +{ + const char *archive_dest = "archive_dest"; + char *val = nullptr; + ObArenaAllocator alloc; + EXPECT_EQ(OB_SUCCESS, deep_copy_cstring(alloc, archive_dest, val)); + EXPECT_STREQ(archive_dest, val); + EXPECT_EQ(OB_ERR_UNEXPECTED, deep_copy_cstring(alloc, archive_dest, val)); + val = nullptr; + EXPECT_EQ(OB_SUCCESS, deep_copy_cstring(alloc, nullptr, val)); + EXPECT_EQ(nullptr, val); + EXPECT_EQ(OB_SUCCESS, deep_copy_cstring(alloc, "", val)); + EXPECT_STREQ("", val); +} +TEST(test_ob_log_miner_utils, ObLogMinerUtilParseLineString) +{ + const char *key_str = "PROGRESS"; + const char *buf = "PROGRESS=abcde\n"; + int64_t pos = 0; + ObString value; + ObArenaAllocator alloc; + char *cstr = nullptr; + EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, value)); + EXPECT_EQ(value, ObString("abcde")); + EXPECT_EQ(pos, strlen(buf)); + pos = 0; + key_str = "CUR_FILE_ID"; + EXPECT_EQ(OB_INVALID_DATA, parse_line(key_str, buf, strlen(buf), pos, value)); + buf = "CUR_FILE_ID=001\n"; + pos = 0; + EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, value)); + EXPECT_EQ(ObString("001"), value); + EXPECT_EQ(pos, strlen(buf)); + pos = 0; + key_str = "kkkk"; + buf = "kkkk=llll\n"; + EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, alloc, cstr)); + EXPECT_STREQ("llll", cstr); + const char *charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwlyz_-."; + const int64_t charset_len = strlen(charset); + char data_buf[100]; + char key[30]; + char val[30]; + for (int i = 0; i < 10000; i++) { + int64_t kv_len = abs(rand()) % (sizeof(key)-1) + 1; + pos = 0; + for (int j = 0; j < kv_len; j++) { + int k = rand() % charset_len; + int v = rand() % charset_len; + key[j] = charset[k]; + val[j] = charset[v]; + } + key[kv_len] = '\0'; + val[kv_len] = '\0'; + snprintf(data_buf, 100, "%s=%s\n", key, val); + EXPECT_EQ(OB_SUCCESS, parse_line(key, data_buf, sizeof(data_buf), pos, value)); + EXPECT_EQ(ObString(val), value); + pos = 0; + EXPECT_EQ(OB_SUCCESS, parse_line(key, data_buf, sizeof(data_buf), pos, alloc, cstr)); + EXPECT_STREQ(val, cstr); + } +} + +TEST(test_ob_log_miner_utils, ObLogMinerUtilsStrToLL) +{ + int64_t num = 0; + EXPECT_EQ(OB_SUCCESS, logminer_str2ll("111222", num)); + EXPECT_EQ(111222, num); + EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll("9223372036854775808", num)); + EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll("-9223372036854775809", num)); + EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll(nullptr, num)); + EXPECT_EQ(OB_SUCCESS, logminer_str2ll("12345aaa", num)); + EXPECT_EQ(12345, num); + const char *timestr = "1688473439\n"; + char *end_ptr = nullptr; + EXPECT_EQ(OB_SUCCESS, logminer_str2ll(timestr, end_ptr, num)); + EXPECT_EQ(1688473439, num); + EXPECT_EQ(end_ptr, timestr + strlen(timestr)-1); + timestr = "aaaaa"; + EXPECT_EQ(OB_INVALID_ARGUMENT, logminer_str2ll(timestr, end_ptr, num)); + EXPECT_EQ(end_ptr, timestr); +} + +TEST(test_ob_log_miner_utils, ObLogMinerUtilsParseLine) +{ + const char *key_str = "PROGRESS"; + const char *buf = "PROGRESS=1234567\n"; + int64_t pos = 0; + int64_t data = 0; + EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, data)); + EXPECT_EQ(data, 1234567); + EXPECT_EQ(pos, strlen(buf)); + pos = 0; + key_str = "CUR_FILE_ID"; + EXPECT_EQ(OB_INVALID_DATA, parse_line(key_str, buf, strlen(buf), pos, data)); + buf = "CUR_FILE_ID=001\n"; + pos = 0; + EXPECT_EQ(OB_SUCCESS, parse_line(key_str, buf, strlen(buf), pos, data)); + EXPECT_EQ(1, data); + EXPECT_EQ(pos, strlen(buf)); + buf = "CUR_FILE_ID:1\n"; + pos = 0; + EXPECT_EQ(OB_INVALID_DATA, parse_line(key_str, buf, strlen(buf), pos, data)); + buf = "CUR_FILE_ID=1"; + pos = 0; + EXPECT_EQ(OB_SIZE_OVERFLOW, parse_line(key_str, buf, strlen(buf), pos, data)); + const char *charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwlyz_-."; + const int64_t charset_len = strlen(charset); + char data_buf[100]; + char key[30]; + for (int i = 0; i < 10000; i++) { + int64_t key_len = abs(rand()) % (sizeof(key)-1) + 1; + pos = 0; + for (int j = 0; j < key_len; j++) { + int k = rand() % charset_len; + key[j] = charset[k]; + } + key[key_len] = '\0'; + int64_t val = rand(); + snprintf(data_buf, 100, "%s=%ld\n", key, val); + EXPECT_EQ(OB_SUCCESS, parse_line(key, data_buf, sizeof(data_buf), pos, data)); + EXPECT_EQ(val, data); + } +} + +TEST(test_ob_log_miner_utils, ParseDigit) +{ + + int64_t pos = 0; + int64_t data = 0; + const char *buf = "12345667"; + EXPECT_EQ(OB_SUCCESS, parse_int(buf, strlen(buf), pos, data)); + EXPECT_EQ(data, 12345667); + EXPECT_EQ(pos, strlen(buf)); + pos = 0; data = 0; + const char buf0[] = {'1','2','3','4','5'}; + EXPECT_EQ(OB_SUCCESS, parse_int(buf0, sizeof(buf0), pos, data)); + EXPECT_EQ(data, 12345); + EXPECT_EQ(pos, sizeof(buf0)); + pos = 0; data = 0; + const char *buf2 = " +1234567aaa"; + EXPECT_EQ(OB_SUCCESS, parse_int(buf2, strlen(buf2), pos, data)); + EXPECT_EQ(data, 1234567); + EXPECT_EQ(pos, 10); + pos = 0; data = 0; + const char *buf3 = " -1234567aaa"; + EXPECT_EQ(OB_SUCCESS, parse_int(buf3, strlen(buf3), pos, data)); + EXPECT_EQ(data, -1234567); + EXPECT_EQ(pos, 10); + pos = 0; data = 0; + const char *buf4 = "9223372036854775807"; + EXPECT_EQ(OB_SUCCESS, parse_int(buf4, strlen(buf4), pos, data)); + EXPECT_EQ(data, 9223372036854775807LL); + EXPECT_EQ(pos, strlen(buf4)); + pos = 0; data = 0; + const char *buf5 = "-9223372036854775808"; + EXPECT_EQ(OB_SUCCESS, parse_int(buf5, strlen(buf5), pos, data)); + EXPECT_EQ(data, -9223372036854775807LL - 1); + EXPECT_EQ(pos, strlen(buf5)); + pos = 0; data = 0; + const char *buf6 = "9223372036854775808"; + EXPECT_EQ(OB_SIZE_OVERFLOW, parse_int(buf6, strlen(buf6), pos, data)); + pos = 0; data = 0; + const char *buf7 = "-9223372036854775809"; + EXPECT_EQ(OB_SIZE_OVERFLOW, parse_int(buf7, strlen(buf7), pos, data)); + pos = 0; data = 0; + const char *buf_invalid = "+-111"; + EXPECT_EQ(OB_INVALID_DATA, parse_int(buf_invalid, strlen(buf_invalid), pos, data)); + pos = 0; data = 0; + buf_invalid = " + 111"; + EXPECT_EQ(OB_INVALID_DATA, parse_int(buf_invalid, strlen(buf_invalid), pos, data)); + pos = 0; data = 0; + buf_invalid = " a111"; + EXPECT_EQ(OB_INVALID_DATA, parse_int(buf_invalid, strlen(buf_invalid), pos, data)); + pos = 0; data = 0; + char buf_tmp[100]; + for (int i = 0; i < 10000; i++) { + pos = 0; + int64_t data1 = rand(); + EXPECT_EQ(OB_SUCCESS, databuff_printf(buf_tmp, 100, pos, "CUR_FILE_ID=")); + int64_t tmp_pos = pos; + EXPECT_EQ(OB_SUCCESS, databuff_printf(buf_tmp, 100, pos, "%ld\n", data1)); + EXPECT_EQ(OB_SUCCESS, parse_int(buf_tmp, 100, tmp_pos, data)); + EXPECT_EQ(data, data1); + } +} + +TEST(test_ob_log_miner_utils, ExpectToken) +{ + const char *buf = "MIN_COMMIT_TS=1"; + int64_t pos = 0; + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "MIN_COMMIT_TS")); + EXPECT_EQ(pos, 13); + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "=")); + EXPECT_EQ(pos, 14); + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "1")); + EXPECT_EQ(pos, 15); + pos = 0; + + buf = "abcdefghxxxx"; + EXPECT_EQ(OB_SIZE_OVERFLOW, expect_token(buf, strlen(buf), pos, "abcdefghxxxxx")); + pos = 0; + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "abcdefgh")); + EXPECT_EQ(pos, 8); + EXPECT_EQ(OB_SIZE_OVERFLOW, expect_token(buf, strlen(buf), pos, "xxxxx")); + pos = 8; + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "xxxx")); + pos = 0; + + buf = "CUR_FILE_ID=1\n"; + EXPECT_EQ(OB_INVALID_DATA, expect_token(buf, strlen(buf), pos, "MAX_FILE_ID")); + pos = 0; + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "CUR_FILE_ID")); + EXPECT_EQ(pos, 11); + EXPECT_EQ(OB_INVALID_DATA, expect_token(buf, strlen(buf), pos, "1")); + pos = 11; + EXPECT_EQ(OB_SUCCESS, expect_token(buf, strlen(buf), pos, "=1\n")); + EXPECT_EQ(pos, 14); + pos = 0; +} + +TEST(test_ob_log_miner_utils, ConvertUintToBit) +{ + uint64_t val = 0; + ObArenaAllocator tmp_alloc; + ObStringBuffer bit_str(&tmp_alloc); + EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str)); + EXPECT_STREQ("0", bit_str.ptr()); + bit_str.reset(); + + val = 1; + EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str)); + EXPECT_STREQ("1", bit_str.ptr()); + bit_str.reset(); + + val = 6; + EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str)); + EXPECT_STREQ("110", bit_str.ptr()); + bit_str.reset(); + + val = 1836032; + EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str)); + EXPECT_STREQ("111000000010000000000", bit_str.ptr()); + bit_str.reset(); + + val = 183848324234; + EXPECT_EQ(OB_SUCCESS, uint_to_bit(val, bit_str)); + EXPECT_STREQ("10101011001110001101101100110010001010", bit_str.ptr()); + bit_str.reset(); +} + +TEST(test_ob_log_miner_utils, ObLogMinerUtilsWriteRecord) +{ + ObConcurrentFIFOAllocator alloc; + ObStringBuffer str_buf(&alloc); + KeyArray key_arr; + EXPECT_EQ(OB_SUCCESS, alloc.init(1 << 20, 1 << 20, 1 << 13)); + EXPECT_EQ(key_arr.push_back("aaa"), OB_SUCCESS); + EXPECT_EQ(key_arr.push_back("bbb"), OB_SUCCESS); + EXPECT_EQ(key_arr.push_back("ccc"), OB_SUCCESS); + EXPECT_EQ(OB_SUCCESS, write_keys(key_arr, str_buf)); + EXPECT_STREQ("\"aaa/bbb/ccc\"", str_buf.ptr()); + str_buf.reset(); + EXPECT_EQ(OB_SUCCESS, write_signed_number(-12345, str_buf)); + EXPECT_STREQ("-12345", str_buf.ptr()); + str_buf.reset(); + EXPECT_EQ(OB_SUCCESS, write_unsigned_number(1111, str_buf)); + EXPECT_STREQ("1111", str_buf.ptr()); + str_buf.reset(); + EXPECT_EQ(OB_SUCCESS, write_string_no_escape("aaaaabbbb'", str_buf)); + EXPECT_STREQ("\"aaaaabbbb'\"", str_buf.ptr()); + str_buf.reset(); +} + +} +} + +int main(int argc, char **argv) +{ + // testing::FLAGS_gtest_filter = "DO_NOT_RUN"; + system("rm -f test_ob_log_miner_utils.log"); + oceanbase::ObLogger &logger = oceanbase::ObLogger::get_logger(); + logger.set_file_name("test_ob_log_miner_utils.log", true, false); + logger.set_log_level("DEBUG"); + logger.set_enable_async_log(false); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +}