database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](unique-key-merge-on-write) update delete bitmap when increamental clone (#12364)

Browse Source
This commit is contained in:
Xin Liao
2022-09-09 17:03:27 +08:00
committed by GitHub
parent 77b93ebc09
commit 554ba40b13
5 changed files with 36 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/olap/memtable.cpp
View File

@ -413,8 +413,9 @@ Status MemTable::_generate_delete_bitmap() {
RETURN_IF_ERROR(beta_rowset->load_segment(beta_rowset->num_segments() - 1, &segment));
segments.push_back(segment);
std::shared_lock meta_rlock(_tablet->get_header_lock());
int64_t end_version = _tablet->max_version_unlocked().second;
RETURN_IF_ERROR(_tablet->calc_delete_bitmap(beta_rowset->rowset_id(), segments, &_rowset_ids,
_delete_bitmap));
_delete_bitmap, end_version));
return Status::OK();
}

3
be/src/olap/rowset/rowset_tree.cpp
View File

@ -200,7 +200,6 @@ void RowsetTree::FindRowsetsWithKeyInRange(
const Slice& encoded_key, const RowsetIdUnorderedSet* rowset_ids,
vector<std::pair<RowsetSharedPtr, int32_t>>* rowsets) const {
DCHECK(initted_);
DCHECK(rowset_ids != nullptr);
// Query the interval tree to efficiently find rowsets with known bounds
// whose ranges overlap the probe key.
@ -209,7 +208,7 @@ void RowsetTree::FindRowsetsWithKeyInRange(
tree_->FindContainingPoint(encoded_key, &from_tree);
rowsets->reserve(rowsets->size() + from_tree.size());
for (RowsetWithBounds* rs : from_tree) {
if (rowset_ids->find(rs->rowset->rowset_id()) != rowset_ids->end()) {
if (!rowset_ids || rowset_ids->find(rs->rowset->rowset_id()) != rowset_ids->end()) {
rowsets->emplace_back(rs->rowset, rs->segment_id);
}
}

32
be/src/olap/tablet.cpp
View File

@ -222,6 +222,10 @@ Status Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>& rowset
auto new_rowset_tree = std::make_unique<RowsetTree>();
ModifyRowSetTree(*_rowset_tree, rs_to_delete, rs_to_add, new_rowset_tree.get());
_rowset_tree = std::move(new_rowset_tree);
for (auto rowset_ptr : rs_to_add) {
RETURN_IF_ERROR(update_delete_bitmap_without_lock(rowset_ptr));
}
save_meta();
}
// reconstruct from tablet meta
@ -1937,10 +1941,11 @@ Status Tablet::_load_rowset_segments(const RowsetSharedPtr& rowset,
Status Tablet::calc_delete_bitmap(RowsetId rowset_id,
const std::vector<segment_v2::SegmentSharedPtr>& segments,
const RowsetIdUnorderedSet* specified_rowset_ids,
DeleteBitmapPtr delete_bitmap, bool check_pre_segments) {
DeleteBitmapPtr delete_bitmap, int64_t end_version,
bool check_pre_segments) {
std::vector<segment_v2::SegmentSharedPtr> pre_segments;
OlapStopWatch watch;
int64_t end_version = max_version_unlocked().second;
Version dummy_version(end_version + 1, end_version + 1);
for (auto& seg : segments) {
seg->load_pk_index_and_bf(); // We need index blocks to iterate
@ -2047,6 +2052,26 @@ void Tablet::_rowset_ids_difference(const RowsetIdUnorderedSet& cur,
}
}
// The caller should hold _rowset_update_lock and _meta_lock lock.
Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) {
int64_t cur_version = rowset->start_version();
std::vector<segment_v2::SegmentSharedPtr> segments;
_load_rowset_segments(rowset, &segments);
DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(tablet_id());
RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, nullptr, delete_bitmap,
cur_version - 1, true));
for (auto iter = delete_bitmap->delete_bitmap.begin();
iter != delete_bitmap->delete_bitmap.end(); ++iter) {
int ret = _tablet_meta->delete_bitmap().set(
{std::get<0>(iter->first), std::get<1>(iter->first), cur_version}, iter->second);
DCHECK(ret == 1);
}
return Status::OK();
}
Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapPtr delete_bitmap,
const RowsetIdUnorderedSet& pre_rowset_ids) {
RowsetIdUnorderedSet cur_rowset_ids;
@ -2068,9 +2093,10 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapP
for (const auto& to_del : rowset_ids_to_del) {
delete_bitmap->remove({to_del, 0, 0}, {to_del, UINT32_MAX, INT64_MAX});
}
int64_t end_version = max_version_unlocked().second;
if (!rowset_ids_to_add.empty()) {
RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, &rowset_ids_to_add,
delete_bitmap, true));
delete_bitmap, end_version, true));
}
// update version without write lock, compaction and publish_txn

4
be/src/olap/tablet.h
View File

@ -329,8 +329,10 @@ public:
Status calc_delete_bitmap(RowsetId rowset_id,
const std::vector<segment_v2::SegmentSharedPtr>& segments,
const RowsetIdUnorderedSet* specified_rowset_ids,
DeleteBitmapPtr delete_bitmap, bool check_pre_segments = false);
DeleteBitmapPtr delete_bitmap, int64_t version,
bool check_pre_segments = false);
Status update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset);
Status update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapPtr delete_bitmap,
const RowsetIdUnorderedSet& pre_rowset_ids);
RowsetIdUnorderedSet all_rs_id() const;

1
be/src/olap/task/engine_clone_task.cpp
View File

@ -456,6 +456,7 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d
tablet->get_cumulative_compaction_lock());
tablet->set_clone_occurred(true);
std::lock_guard<std::mutex> push_lock(tablet->get_push_lock());
std::lock_guard<std::mutex> rwlock(tablet->get_rowset_update_lock());
std::lock_guard<std::shared_mutex> wrlock(tablet->get_header_lock());
// check clone dir existed
if (!FileUtils::check_exist(clone_dir)) {