database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[chore](binlog) Log download binlog cost (#47360) (#47575)

Browse Source 
cherry-pick #47360
This commit is contained in:
walter
2025-02-08 16:22:23 +08:00
committed by GitHub
parent b4f00be528
commit e681c055a4
1 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
be/src/service/backend_service.cpp
View File

@ -107,10 +107,23 @@ void _ingest_binlog(IngestBinlogArg* arg) {
auto& request = arg->request;
MonotonicStopWatch watch;
watch.start();
int64_t total_download_bytes = 0;
int64_t total_download_files = 0;
TStatus tstatus;
std::vector<std::string> download_success_files;
Defer defer {[=, &tstatus, ingest_binlog_tstatus = arg->tstatus]() {
LOG(INFO) << "ingest binlog. result: " << apache::thrift::ThriftDebugString(tstatus);
Defer defer {[=, &tstatus, ingest_binlog_tstatus = arg->tstatus, &watch, &total_download_bytes,
&total_download_files]() {
auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time() / 1000000);
double copy_rate = 0.0;
if (elapsed_time_ms > 0) {
copy_rate = total_download_bytes / ((double)elapsed_time_ms) / 1000;
}
LOG(INFO) << "ingest binlog elapsed " << elapsed_time_ms << " ms, download "
<< total_download_files << " files, total " << total_download_bytes
<< " bytes, avg rate " << copy_rate
<< " MB/s. result: " << apache::thrift::ThriftDebugString(tstatus);
if (tstatus.status_code != TStatusCode::OK) {
// abort txn
StorageEngine::instance()->txn_manager()->abort_txn(partition_id, txn_id,
@ -263,6 +276,8 @@ void _ingest_binlog(IngestBinlogArg* arg) {
status.to_thrift(&tstatus);
return;
}
total_download_bytes = total_size;
total_download_files = num_segments;
// Step 5.3: get all segment files
for (int64_t segment_index = 0; segment_index < num_segments; ++segment_index) {
@ -435,6 +450,8 @@ void _ingest_binlog(IngestBinlogArg* arg) {
status.to_thrift(&tstatus);
return;
}
total_download_bytes += total_index_size;
total_download_files += segment_index_file_urls.size();
// Step 6.3: get all segment index files
DCHECK(segment_index_file_sizes.size() == segment_index_file_names.size());