database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhencement](RowsetWriter) Don't delete files when beta rowset writer destructed (#25578)

Browse Source
This commit is contained in:
AlexYue
2023-10-19 09:37:04 +08:00
committed by GitHub
parent 45178f5664
commit 63c89df474
2 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
be/src/olap/rowset/beta_rowset_writer.cpp
View File

@ -20,6 +20,7 @@
#include <assert.h>
// IWYU pragma: no_include <bthread/errno.h>
#include <errno.h> // IWYU pragma: keep
#include <fmt/format.h>
#include <stdio.h>
#include <ctime> // time
@ -33,7 +34,6 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
#include "gutil/strings/substitute.h"
#include "io/fs/file_reader.h"
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
@ -85,8 +85,8 @@ BetaRowsetWriter::~BetaRowsetWriter() {
if (!_already_built) { // abnormal exit, remove all files generated
WARN_IF_ERROR(_segment_creator.close(),
"close segment creator failed"); // ensure all files are closed
auto fs = _rowset_meta->fs();
if (fs->type() != io::FileSystemType::LOCAL) { // Remote fs will delete them asynchronously
const auto& fs = _rowset_meta->fs();
if (!fs || !_rowset_meta->is_local()) { // Remote fs will delete them asynchronously
return;
}
for (int i = _segment_start_id; i < _segment_creator.next_segment_id(); ++i) {
@ -96,7 +96,7 @@ BetaRowsetWriter::~BetaRowsetWriter() {
// will be cleaned up by the GC background. So here we only print the error
// message when we encounter an error.
WARN_IF_ERROR(fs->delete_file(seg_path),
strings::Substitute("Failed to delete file=$0", seg_path));
fmt::format("Failed to delete file={}", seg_path));
}
}
}

12
be/src/olap/rowset/vertical_beta_rowset_writer.cpp
View File

@ -17,6 +17,7 @@
#include "olap/rowset/vertical_beta_rowset_writer.h"
#include <fmt/format.h>
#include <gen_cpp/olap_file.pb.h>
#include <algorithm>
@ -29,7 +30,6 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/logging.h"
#include "gutil/strings/substitute.h"
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
#include "olap/rowset/beta_rowset.h"
@ -44,8 +44,8 @@ using namespace ErrorCode;
VerticalBetaRowsetWriter::~VerticalBetaRowsetWriter() {
if (!_already_built) {
auto fs = _rowset_meta->fs();
if (!fs) {
const auto& fs = _rowset_meta->fs();
if (!fs || !_rowset_meta->is_local()) { // Remote fs will delete them asynchronously
return;
}
for (auto& segment_writer : _segment_writers) {
@ -56,8 +56,7 @@ VerticalBetaRowsetWriter::~VerticalBetaRowsetWriter() {
// Even if an error is encountered, these files that have not been cleaned up
// will be cleaned up by the GC background. So here we only print the error
// message when we encounter an error.
WARN_IF_ERROR(fs->delete_file(path),
strings::Substitute("Failed to delete file=$0", path));
WARN_IF_ERROR(fs->delete_file(path), fmt::format("Failed to delete file={}", path));
}
}
}
@ -105,7 +104,8 @@ Status VerticalBetaRowsetWriter::add_columns(const vectorized::Block* block,
RETURN_IF_ERROR(_segment_writers[_cur_writer_idx]->init(col_ids, is_key));
}
// when splitting segment, need to make rows align between key columns and value columns
size_t start_offset = 0, limit = num_rows;
size_t start_offset = 0;
size_t limit = num_rows;
if (num_rows_written + num_rows >= num_rows_key_group &&
_cur_writer_idx < _segment_writers.size() - 1) {
RETURN_IF_ERROR(_segment_writers[_cur_writer_idx]->append_block(