database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Compaction][Bug-Fix] Fix bug that meta lock need to be held when calculating compaction score (#4829)

Browse Source 
* [Compaction][Buf] Fix bug that meta lock need to be held when calucating compaction score

* fix

Co-authored-by: morningman <chenmingyu@baidu.com>
This commit is contained in:
Mingyu Chen
2020-11-05 20:29:01 +08:00
committed by GitHub
parent c8df76a807
commit f239f44b37
6 changed files with 15 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/olap/base_compaction.cpp
View File

@ -45,7 +45,7 @@ OLAPStatus BaseCompaction::compact() {
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
// 2. do base compaction, merge rowsets
int64_t permits = _tablet->calc_base_compaction_score();
int64_t permits = _tablet->calc_compaction_score(CompactionType::BASE_COMPACTION);
RETURN_NOT_OK(do_compaction(permits));
TRACE("compaction finished");

2
be/src/olap/cumulative_compaction.cpp
View File

@ -53,7 +53,7 @@ OLAPStatus CumulativeCompaction::compact() {
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
// 3. do cumulative compaction, merge rowsets
int64_t permits = _tablet->calc_cumulative_compaction_score();
int64_t permits = _tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
RETURN_NOT_OK(do_compaction(permits));
TRACE("compaction finished");

10
be/src/olap/tablet.cpp
View File

@ -751,22 +751,24 @@ bool Tablet::can_do_compaction() {
}
const uint32_t Tablet::calc_compaction_score(CompactionType compaction_type) const {
// Need meta lock, because it will iterator "all_rs_metas" of tablet meta.
ReadLock rdlock(&_meta_lock);
if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
return calc_cumulative_compaction_score();
return _calc_cumulative_compaction_score();
} else {
DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION);
return calc_base_compaction_score();
return _calc_base_compaction_score();
}
}
const uint32_t Tablet::calc_cumulative_compaction_score() const {
const uint32_t Tablet::_calc_cumulative_compaction_score() const {
uint32_t score = 0;
_cumulative_compaction_policy->calc_cumulative_compaction_score(
_tablet_meta->all_rs_metas(), cumulative_layer_point(), &score);
return score;
}
const uint32_t Tablet::calc_base_compaction_score() const {
const uint32_t Tablet::_calc_base_compaction_score() const {
uint32_t score = 0;
const int64_t point = cumulative_layer_point();
bool base_rowset_exist = false;

5
be/src/olap/tablet.h
View File

@ -164,8 +164,6 @@ public:
// operation for compaction
bool can_do_compaction();
const uint32_t calc_compaction_score(CompactionType compaction_type) const;
const uint32_t calc_cumulative_compaction_score() const;
const uint32_t calc_base_compaction_score() const;
static void compute_version_hash_from_rowsets(const std::vector<RowsetSharedPtr>& rowsets,
VersionHash* version_hash);
@ -249,6 +247,9 @@ private:
OLAPStatus _capture_consistent_rowsets_unlocked(const vector<Version>& version_path,
vector<RowsetSharedPtr>* rowsets) const;
const uint32_t _calc_cumulative_compaction_score() const;
const uint32_t _calc_base_compaction_score() const;
public:
static const int64_t K_INVALID_CUMULATIVE_POINT = -1;

10
be/src/olap/tablet_manager.cpp
View File

@ -745,15 +745,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
}
}
uint32_t table_score = 0;
{
ReadLock rdlock(tablet_ptr->get_header_lock_ptr());
if (compaction_type == CompactionType::BASE_COMPACTION) {
table_score = tablet_ptr->calc_base_compaction_score();
} else if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
table_score = tablet_ptr->calc_cumulative_compaction_score();
}
}
uint32_t table_score = tablet_ptr->calc_compaction_score(compaction_type);
if (table_score > highest_score) {
highest_score = table_score;
best_tablet = tablet_ptr;

6
be/test/olap/cumulative_compaction_policy_test.cpp
View File

@ -212,7 +212,7 @@ TEST_F(TestNumBasedCumulativeCompactionPolicy, calc_cumulative_compaction_score)
TabletSharedPtr _tablet(new Tablet(_tablet_meta, nullptr, CUMULATIVE_NUM_BASED_POLICY));
_tablet->init();
const uint32_t score = _tablet->calc_cumulative_compaction_score();
const uint32_t score = _tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
ASSERT_EQ(15, score);
}
@ -675,7 +675,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, calc_cumulative_compaction_score
_tablet->init();
_tablet->calculate_cumulative_point();
const uint32_t score = _tablet->calc_cumulative_compaction_score();
const uint32_t score = _tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
ASSERT_EQ(15, score);
}
@ -692,7 +692,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, calc_cumulative_compaction_score
TabletSharedPtr _tablet(new Tablet(_tablet_meta, nullptr, CUMULATIVE_SIZE_BASED_POLICY));
_tablet->init();
_tablet->calculate_cumulative_point();
const uint32_t score = _tablet->calc_cumulative_compaction_score();
const uint32_t score = _tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION);
ASSERT_EQ(7, score);
}