diff --git a/cmake/RPM.cmake b/cmake/RPM.cmake index 21cd8fcc6..75b75e956 100644 --- a/cmake/RPM.cmake +++ b/cmake/RPM.cmake @@ -119,6 +119,12 @@ install( DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT cdc ) +install( + DIRECTORY + ${PROJECT_SOURCE_DIR}/src/logservice/libobcdc/tests/scripts/ + DESTINATION ${CMAKE_INSTALL_RUNSTATEDIR} + COMPONENT cdc + ) if(OB_BUILD_OPENSOURCE) install( diff --git a/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp b/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp index c8b0f461a..2b0c87e26 100644 --- a/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp +++ b/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp @@ -136,7 +136,7 @@ int ObCdcStartLsnLocator::do_req_start_lsn_(const ObLocateLSNByTsReq &req, ret = OB_IN_STOP_STATE; } - EXTLOG_LOG(INFO, "locator req success", KR(ret), K(tenant_id_), K(req), K(resp), K(is_hurry_quit)); + EXTLOG_LOG(INFO, "locator req done", KR(ret), K(tenant_id_), K(req), K(resp), K(is_hurry_quit)); } return ret; @@ -185,7 +185,7 @@ int ObCdcStartLsnLocator::do_locate_ls_(const bool fetch_archive_only, need_seek_archive = true; } else { if (OB_FAIL(init_palf_handle_guard_(ls_id, palf_handle_guard))) { - if (OB_LS_NOT_EXIST == ret) { + if (OB_LS_NOT_EXIST == ret && OB_SYS_TENANT_ID != tenant_id_) { ret = OB_SUCCESS; need_seek_archive = true; } else { @@ -195,7 +195,7 @@ int ObCdcStartLsnLocator::do_locate_ls_(const bool fetch_archive_only, // - OB_ENTRY_NOT_EXIST: there is no log's log_ts is higher than ts_ns // - OB_ERR_OUT_OF_LOWER_BOUND: ts_ns is too old, log files may have been recycled } else if (OB_FAIL(palf_handle_guard.seek(start_scn, group_iter))) { - if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { + if (OB_ERR_OUT_OF_LOWER_BOUND == ret && OB_SYS_TENANT_ID != tenant_id_) { ret = OB_SUCCESS; need_seek_archive = true; } else { @@ -208,10 +208,6 @@ int ObCdcStartLsnLocator::do_locate_ls_(const bool fetch_archive_only, } else { result_ts_ns = log_group_entry.get_scn().get_val_for_logservice(); } - - if (OB_SYS_TENANT_ID == tenant_id_) { - need_seek_archive = false; - } // Note: us } diff --git a/src/logservice/libobcdc/src/ob_cdc_auto_config_mgr.cpp b/src/logservice/libobcdc/src/ob_cdc_auto_config_mgr.cpp index f3db57286..21a6fe6a2 100644 --- a/src/logservice/libobcdc/src/ob_cdc_auto_config_mgr.cpp +++ b/src/logservice/libobcdc/src/ob_cdc_auto_config_mgr.cpp @@ -92,7 +92,7 @@ void ObCDCAutoConfigMgr::init_queue_length_(const ObLogConfig &config) const int64_t msg_sorter_queue_length = br_queue_length; REFRESH_NUM_FIELD_WITH_CONFIG(msg_sorter_task_count_upper_limit, msg_sorter_queue_length, config.msg_sorter_task_count_upper_limit.get()); - REFRESH_NUM_FIELD_WITH_CONFIG(sequencer_queue_length, MAX_QUEUE_LENGTH, config.sequencer_queue_length.get()); + REFRESH_NUM_FIELD_WITH_CONFIG(sequencer_queue_length, MAX_QUEUE_LENGTH / 10, config.sequencer_queue_length.get()); REFRESH_NUM_FIELD_WITH_CONFIG(storager_queue_length, DEFAULT_STORAGE_QUEUE_LENGTH, config.storager_queue_length.get()); REFRESH_NUM_FIELD_WITH_CONFIG(reader_queue_length, DEFAULT_STORAGE_QUEUE_LENGTH, config.reader_queue_length.get()); } diff --git a/src/logservice/libobcdc/tests/scripts/kill_obcdc.sh b/src/logservice/libobcdc/tests/scripts/kill_obcdc.sh new file mode 100755 index 000000000..c97186af9 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/kill_obcdc.sh @@ -0,0 +1 @@ +cat run/libobcdc.pid | xargs kill -9 diff --git a/src/logservice/libobcdc/tests/scripts/libobcdc.conf b/src/logservice/libobcdc/tests/scripts/libobcdc.conf new file mode 100644 index 000000000..449612f84 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/libobcdc.conf @@ -0,0 +1,33 @@ +######### [cluster_info] ######### +cluster_db_name=oceanbase +cluster_password= +cluster_user= +cluster_url= + +######### [working_mode] ######### +#working_mode=memory +meta_data_refresh_mode=data_dict +#meta_data_refresh_mode=online + +######### [synchronous information] ######### +tb_white_list=*.*.* +#tb_white_list=sys.oblog*.*|oblog_tt.*.* +tb_black_list=*.*.*_t|*.*.*_[0-9][a-z] + + +######### [log] ######### +log_level=ALL.*:INFO;PALF.*:WARN;SHARE.SCHEMA:WARN;TLOG.FETCHER:INFO;TLOG.FORMATTER:INFO; +#log_level=ALL.*:DEBUG; + +skip_dirty_data=0 +skip_ob_version_compat_check=1 +#enable_output_hidden_primary_key=1 +#print_participant_not_serve_info=1 + +mysql_connect_timeout_sec=40 +mysql_query_timeout_sec=30 + +######### [flow controll] ######### +memory_limit=8G +pause_redo_dispatch_task_count_threshold=80 +memory_usage_warn_threshold=85 diff --git a/src/logservice/libobcdc/tests/scripts/start.sh b/src/logservice/libobcdc/tests/scripts/start.sh new file mode 100755 index 000000000..78e958c17 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/start.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +TIMESTAMP=`date -d "2023-6-16 19:30:31" +%s` +TIMESTAMP=0 + +CONFIG=conf/libobcdc.conf +DATA_FILE=data/data.log + +RUN_TIME=60 + +LOG_DIR=./log + +./kill_obcdc.sh + +rm -fr $LOG_DIR/ core.* ${DATA_FILE}* + +export LD_LIBRARY_PATH=./lib/:$LD_LIBRARY_PATH +./obcdc_tailf -v +ulimit -c unlimited +## work in background +`pwd`/obcdc_tailf -f $CONFIG -t $TIMESTAMP -d +#`pwd`/obcdc_tailf -f $CONFIG -T $TIMESTAMP -d +#./obcdc_tailf_static -f $CONFIG -T $TIMESTAMP -d +#`pwd`/obcdc_tailf -f $CONFIG -T $TIMESTAMP -D${DATA_FILE} -d + +# Timed runs in the background +# `pwd`/obcdc_tailf -R $RUN_TIME -f $CONFIG -t $TIMESTAMP -d + +# output data +# `pwd`/obcdc_tailf -V -f $CONFIG -T $TIMESTAMP -D${DATA_FILE} 2>&1 | grep -v tid + +# Timed runs with output data +# `pwd`/obcdc_tailf -x -o -R$RUN_TIME -f $CONFIG -t $TIMESTAMP -D${DATA_FILE} 2>&1 | grep -v tid diff --git a/src/logservice/libobcdc/tests/scripts/top.sh b/src/logservice/libobcdc/tests/scripts/top.sh new file mode 100755 index 000000000..8a36bb3a8 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/top.sh @@ -0,0 +1,3 @@ +PID=`cat run/libobcdc.pid` + +top -d 1 -p $PID diff --git a/src/logservice/libobcdc/tests/scripts/watch_delay.sh b/src/logservice/libobcdc/tests/scripts/watch_delay.sh new file mode 100755 index 000000000..458078fd6 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/watch_delay.sh @@ -0,0 +1,4 @@ + +LOG_FILE=log/libobcdc.log + +watch -n 1 "if [ -f $LOG_FILE ]; then grep MIN_DELAY $LOG_FILE | grep HEARTBEAT | awk '{print \$11, \$12, \$13}' | tail -n 3; fi" diff --git a/src/logservice/libobcdc/tests/scripts/watch_tps.sh b/src/logservice/libobcdc/tests/scripts/watch_tps.sh new file mode 100755 index 000000000..226393ae7 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/watch_tps.sh @@ -0,0 +1,4 @@ + +LOG_FILE=log/libobcdc.log + +watch -n 1 "if [ -f $LOG_FILE ]; then grep NEXT_RECORD_TPS $LOG_FILE | awk '{print \$1, \$2, \$10, \$11, \$12, \$13}' | tail -n 3; fi" diff --git a/src/logservice/libobcdc/tests/scripts/watch_traffic.sh b/src/logservice/libobcdc/tests/scripts/watch_traffic.sh new file mode 100755 index 000000000..7c9abf504 --- /dev/null +++ b/src/logservice/libobcdc/tests/scripts/watch_traffic.sh @@ -0,0 +1,3 @@ +LOG=log/libobcdc.log + +watch -n 1 "if [ -f $LOG ]; then grep traffic $LOG | grep -v traffic=0.00B | awk '{printf(\"%-26s %-13s %-10s %-10s %-14s %-12s %-18s %-28s %-25s %-26s\n\", \$10, \$11, \$12, \$13, \$14, \$15, \$17, \$18, \$19, \$20);}' | tail -n 3; fi"