database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Fix BE do_tablet_meta_checkpoint retain _meta_lock for a long time (#2430)

Browse Source 
Add a flag in RowsetMeta to record whether it has been deleted from rowset meta.
Before this PR, 37156 rowsets only cost 1642 s.
With this PR, 37319 rowsets just cost 1 s.
This commit is contained in:
Lijia Liu
2019-12-12 23:21:43 +08:00
committed by ZHAO Chun
parent 59f5851c29
commit 4d958ec7a1
3 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
be/src/olap/rowset/rowset_meta.h
View File

@ -296,6 +296,19 @@ public:
return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version();
}
// Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta.
// But, this check behavior may cost a lot of time when it is frequent.
// If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true,
// And next time when we want to check if this rowset is in rowset mata manager's meta, we can
// check is_remove_from_rowset_meta() first.
void set_remove_from_rowset_meta() {
_is_removed_from_rowset_meta = true;
}
bool is_remove_from_rowset_meta() const {
return _is_removed_from_rowset_meta;
}
private:
friend class AlphaRowsetMeta;
bool _deserialize_from_pb(const std::string& value) {
@ -332,6 +345,7 @@ private:
private:
RowsetMetaPB _rowset_meta_pb;
RowsetId _rowset_id;
bool _is_removed_from_rowset_meta = false;
};
} // namespace doris

2
be/src/olap/storage_engine.cpp
View File

@ -1036,7 +1036,7 @@ void* StorageEngine::_tablet_checkpoint_callback(void* arg) {
#endif
LOG(INFO) << "try to start tablet meta checkpoint thread!";
while (true) {
LOG(INFO) << "begin to do tablet meta checkpoint";
LOG(INFO) << "begin to do tablet meta checkpoint:" << ((DataDir*)arg)->path();
int64_t start_time = UnixMillis();
_tablet_manager->do_tablet_meta_checkpoint((DataDir*)arg);
int64_t used_time = (UnixMillis() - start_time) / 1000;

7
be/src/olap/tablet.cpp
View File

@ -1056,12 +1056,17 @@ OLAPStatus Tablet::do_tablet_meta_checkpoint() {
RETURN_NOT_OK(save_meta());
// if save meta successfully, then should remove the rowset meta existing in tablet
// meta from rowset meta store
for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
// If we delete it from rowset manager's meta explicitly in previous checkpoint, just skip.
if(rs_meta->is_remove_from_rowset_meta()) {
continue;
}
if (RowsetMetaManager::check_rowset_meta(_data_dir->get_meta(), tablet_uid(), rs_meta->rowset_id())) {
RowsetMetaManager::remove(_data_dir->get_meta(), tablet_uid(), rs_meta->rowset_id());
LOG(INFO) << "remove rowset id from meta store because it is already persistent with tablet meta"
<< ", rowset_id=" << rs_meta->rowset_id();
}
rs_meta->set_remove_from_rowset_meta();
}
_newly_created_rowset_num = 0;
_last_checkpoint_time = UnixMillis();