database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](fd) Recycle the segment file fds directly when delete stale rowset (#22705)

Browse Source
This commit is contained in:
Lightman
2023-08-09 14:45:56 +08:00
committed by GitHub
parent 19a2617d70
commit 1d1077c3b6
6 changed files with 8 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/olap/olap_server.cpp
View File

@ -249,9 +249,8 @@ Status StorageEngine::start_bg_threads() {
}
void StorageEngine::_cache_clean_callback() {
int32_t interval = 600;
int32_t interval = config::cache_prune_stale_interval;
while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
interval = config::cache_prune_stale_interval;
if (interval <= 0) {
LOG(WARNING) << "config of cache clean interval is illegal: [" << interval
<< "], force set to 3600 ";

2
be/src/olap/segment_loader.cpp
View File

@ -87,7 +87,7 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
return Status::OK();
}
void SegmentLoader::erase_segment(const SegmentCache::CacheKey& key) {
void SegmentLoader::erase_segments(const SegmentCache::CacheKey& key) {
_segment_cache->erase(key);
}

2
be/src/olap/segment_loader.h
View File

@ -111,7 +111,7 @@ public:
Status load_segments(const BetaRowsetSharedPtr& rowset, SegmentCacheHandle* cache_handle,
bool use_cache = false);
void erase_segment(const SegmentCache::CacheKey& key);
void erase_segments(const SegmentCache::CacheKey& key);
private:
SegmentLoader();

2
be/src/olap/tablet.cpp
View File

@ -665,6 +665,8 @@ void Tablet::_delete_stale_rowset_by_version(const Version& version) {
return;
}
_tablet_meta->delete_stale_rs_meta_by_version(version);
// If the stale rowset was deleted, it need to remove the fds directly
SegmentLoader::instance()->erase_segments(SegmentCache::CacheKey(rowset_meta->rowset_id()));
VLOG_NOTICE << "delete stale rowset. tablet=" << full_name() << ", version=" << version;
}

2
be/src/olap/txn_manager.cpp
View File

@ -378,7 +378,7 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id,
rowset->merge_rowset_meta(transient_rowset->rowset_meta());
// erase segment cache cause we will add a segment to rowset
SegmentLoader::instance()->erase_segment(rowset->rowset_id());
SegmentLoader::instance()->erase_segments(rowset->rowset_id());
}
stats->partial_update_write_segment_us = MonotonicMicros() - t3;
int64_t t4 = MonotonicMicros();

3
be/src/vec/olap/vertical_block_reader.cpp
View File

@ -69,8 +69,9 @@ Status VerticalBlockReader::_get_segment_iterators(const ReaderParams& read_para
// segment iterator will be inited here
// In vertical compaction, every group will load segment so we should cache
// segment to avoid tot many s3 head request
bool use_cache = !rs_split.rs_reader->rowset()->is_local();
RETURN_IF_ERROR(rs_split.rs_reader->get_segment_iterators(&_reader_context, segment_iters,
{}, true));
{}, use_cache));
// if segments overlapping, all segment iterator should be inited in
// heap merge iterator. If segments are none overlapping, only first segment of this
// rowset will be inited and push to heap, other segment will be inited later when current