database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Some refactors on TabletManager (#2918)

Browse Source 
1. Add some comments to make the code easier to understand;
2. Make the metric `create_tablet_requests_failed` to be accurate;
3. Some internal methods use naked pointers directly instead of `shared_ptr`;
4. The `using` in `.h` files are contagious when included by other files,
    so we should only use it in `.cpp` files;
5. Some formatting changes: such as wrapping lines that are too long
6. Parameters that need to be modified, use pointers instead of references

No functional changes in this patch.
This commit is contained in:
LingBin
2020-02-17 14:50:29 +08:00
committed by GitHub
parent f20eb12457
commit feef077520
9 changed files with 573 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
be/src/olap/tablet.cpp
View File

@ -48,16 +48,9 @@ using std::sort;
using std::string;
using std::vector;
TabletSharedPtr Tablet::create_tablet_from_meta(
TabletMetaSharedPtr tablet_meta,
DataDir* data_dir) {
TabletSharedPtr tablet = std::make_shared<Tablet>(tablet_meta, data_dir);
if (tablet == nullptr) {
LOG(WARNING) << "fail to malloc a table.";
return nullptr;
}
return tablet;
TabletSharedPtr Tablet::create_tablet_from_meta(TabletMetaSharedPtr tablet_meta,
DataDir* data_dir) {
return std::make_shared<Tablet>(tablet_meta, data_dir);
}
void Tablet::_gen_tablet_path() {
@ -535,6 +528,8 @@ OLAPStatus Tablet::add_delete_predicate(
return _tablet_meta->add_delete_predicate(delete_predicate, version);
}
// TODO(lingbin): what is the difference between version_for_delete_predicate() and
// version_for_load_deletion()? should at least leave a comment
bool Tablet::version_for_delete_predicate(const Version& version) {
return _tablet_meta->version_for_delete_predicate(version);
}

6
be/src/olap/tablet.h
View File

@ -47,9 +47,8 @@ using TabletSharedPtr = std::shared_ptr<Tablet>;
class Tablet : public std::enable_shared_from_this<Tablet> {
public:
static TabletSharedPtr create_tablet_from_meta(
TabletMetaSharedPtr tablet_meta,
DataDir* data_dir = nullptr);
static TabletSharedPtr create_tablet_from_meta(TabletMetaSharedPtr tablet_meta,
DataDir* data_dir = nullptr);
Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir);
~Tablet();
@ -241,7 +240,6 @@ public:
bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta);
void build_tablet_report_info(TTabletInfo* tablet_info);
OLAPStatus generate_tablet_meta_copy(TabletMetaSharedPtr new_tablet_meta);

1037
be/src/olap/tablet_manager.cpp
View File

File diff suppressed because it is too large Load Diff

61
be/src/olap/tablet_manager.h
View File

@ -24,6 +24,7 @@
#include <set>
#include <string>
#include <vector>
#include <unordered_map>
#include "agent/status.h"
#include "common/status.h"
@ -50,11 +51,14 @@ public:
bool check_tablet_id_exist(TTabletId tablet_id);
void clear();
// Create new tablet for StorageEngine
OLAPStatus create_tablet(const TCreateTabletReq& request,
std::vector<DataDir*> stores);
// The param stores holds all candidate data_dirs for this tablet.
// NOTE: If the request is from a schema-changing tablet, The directory selected by the
// new tablet should be the same as the directory of origin tablet. Because the
// linked-schema-change type requires Linux hard-link, which does not support cross disk.
// TODO(lingbin): Other schema-change type do not need to be on the same disk. Because
// there may be insufficient space on the current disk, which will lead the schema-change
// task to be fail, even if there is enough space on other disks
OLAPStatus create_tablet(const TCreateTabletReq& request, std::vector<DataDir*> stores);
// Drop a tablet by description
// If set keep_files == true, files will NOT be deleted when deconstruction.
@ -89,7 +93,7 @@ public:
static bool get_rowset_id_from_path(const std::string& path, RowsetId* rowset_id);
void get_tablet_stat(TTabletStatResult& result);
void get_tablet_stat(TTabletStatResult* result);
// parse tablet header msg to generate tablet object
OLAPStatus load_tablet_from_meta(DataDir* data_dir, TTabletId tablet_id,
@ -138,7 +142,9 @@ private:
bool keep_files, bool drop_old);
bool _check_tablet_id_exist_unlocked(TTabletId tablet_id);
OLAPStatus _create_inital_rowset_unlocked(TabletSharedPtr tablet, const TCreateTabletReq& request);
OLAPStatus _create_inital_rowset_unlocked(const TCreateTabletReq& request,
Tablet* tablet);
OLAPStatus _drop_tablet_directly_unlocked(TTabletId tablet_id,
TSchemaHash schema_hash,
bool keep_files = false);
@ -151,17 +157,17 @@ private:
TabletSharedPtr _internal_create_tablet_unlocked(const AlterTabletType alter_type,
const TCreateTabletReq& request,
const bool is_schema_change_tablet,
const TabletSharedPtr ref_tablet,
std::vector<DataDir*> data_dirs);
const bool is_schema_change,
const Tablet* base_tablet,
const std::vector<DataDir*>& data_dirs);
TabletSharedPtr _create_tablet_meta_and_dir_unlocked(const TCreateTabletReq& request,
const bool is_schema_change_tablet,
const TabletSharedPtr ref_tablet,
std::vector<DataDir*> data_dirs);
const bool is_schema_change,
const Tablet* base_tablet,
const std::vector<DataDir*>& data_dirs);
OLAPStatus _create_tablet_meta_unlocked(const TCreateTabletReq& request,
DataDir* store,
const bool is_schema_change_tablet,
const TabletSharedPtr ref_tablet,
const Tablet* base_tablet,
TabletMetaSharedPtr* tablet_meta);
void _build_tablet_stat();
@ -169,32 +175,33 @@ private:
void _remove_tablet_from_partition_unlocked(const Tablet& tablet);
private:
DISALLOW_COPY_AND_ASSIGN(TabletManager);
// TODO(lingbin): should be TabletInstances?
// should be removed after schema_hash be removed
struct TableInstances {
Mutex schema_change_lock;
// The first element(i.e. tablet_arr[0]) is the base tablet. When we add new tablet
// to tablet_arr, we will sort all the elements in create-time ascending order,
// which will ensure the first one is base-tablet
std::list<TabletSharedPtr> table_arr;
};
typedef std::map<int64_t, TableInstances> tablet_map_t;
// tablet_id -> TabletInstances
typedef std::unordered_map<int64_t, TableInstances> tablet_map_t;
// Protect _tablet_map, _partition_tablet_map, _shutdown_tablets
RWMutex _tablet_map_lock;
tablet_map_t _tablet_map;
std::map<std::string, DataDir*> _store_map;
// partition_id => tablet_info
std::map<int64_t, std::set<TabletInfo>> _partition_tablet_map;
std::vector<TabletSharedPtr> _shutdown_tablets;
std::mutex _tablet_stat_mutex;
// cache to save tablets' statistics, such as data size and row
// cache to save tablets' statistics, such as data-size and row-count
// TODO(cmy): for now, this is a naive implementation
std::map<int64_t, TTabletStat> _tablet_stat_cache;
// last update time of tablet stat cache
int64_t _tablet_stat_cache_update_time_ms;
uint32_t _available_storage_medium_type_count;
std::vector<TabletSharedPtr> _shutdown_tablets;
// map from partition id to tablet_id
std::map<int64_t, std::set<TabletInfo>> _partition_tablet_map;
DISALLOW_COPY_AND_ASSIGN(TabletManager);
int64_t _last_update_stat_ms;
};
} // namespace doris

34
be/src/olap/tablet_meta.cpp
View File

@ -28,6 +28,10 @@
#include "util/uid_util.h"
#include "util/url_coding.h"
using std::string;
using std::unordered_map;
using std::vector;
namespace doris {
OLAPStatus AlterTabletTask::init_from_pb(const AlterTabletPB& alter_task) {
@ -60,12 +64,23 @@ OLAPStatus TabletMeta::create(int64_t table_id, int64_t partition_id,
int64_t tablet_id, int32_t schema_hash,
uint64_t shard_id, const TTabletSchema& tablet_schema,
uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
const unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletMetaSharedPtr* tablet_meta, TabletUid& tablet_uid) {
tablet_meta->reset(new TabletMeta(table_id, partition_id,
tablet_id, schema_hash,
shard_id, tablet_schema,
next_unique_id, col_ordinal_to_unique_id, tablet_uid));
tablet_id, schema_hash,
shard_id, tablet_schema,
next_unique_id, col_ordinal_to_unique_id, tablet_uid));
return OLAP_SUCCESS;
}
OLAPStatus TabletMeta::create(const TCreateTabletReq& request, const TabletUid& tablet_uid,
uint64_t shard_id, uint32_t next_unique_id,
const unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletMetaSharedPtr* tablet_meta) {
tablet_meta->reset(new TabletMeta(request.table_id, request.partition_id,
request.tablet_id, request.tablet_schema.schema_hash,
shard_id, request.tablet_schema,
next_unique_id, col_ordinal_to_unique_id, tablet_uid));
return OLAP_SUCCESS;
}
@ -175,7 +190,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id,
init_from_pb(tablet_meta_pb);
}
OLAPStatus TabletMeta::create_from_file(const std::string& file_path) {
OLAPStatus TabletMeta::create_from_file(const string& file_path) {
FileHeader<TabletMetaPB> file_header;
FileHandler file_handler;
@ -201,7 +216,7 @@ OLAPStatus TabletMeta::create_from_file(const std::string& file_path) {
return init_from_pb(tablet_meta_pb);
}
OLAPStatus TabletMeta::reset_tablet_uid(const std::string& file_path) {
OLAPStatus TabletMeta::reset_tablet_uid(const string& file_path) {
OLAPStatus res = OLAP_SUCCESS;
TabletMeta tmp_tablet_meta;
if ((res = tmp_tablet_meta.create_from_file(file_path)) != OLAP_SUCCESS) {
@ -226,7 +241,8 @@ OLAPStatus TabletMeta::reset_tablet_uid(const std::string& file_path) {
return res;
}
std::string TabletMeta::construct_header_file_path(const std::string& schema_hash_path, const int64_t tablet_id) {
string TabletMeta::construct_header_file_path(const string& schema_hash_path,
const int64_t tablet_id) {
std::stringstream header_name_stream;
header_name_stream << schema_hash_path << "/" << tablet_id << ".hdr";
return header_name_stream.str();
@ -429,7 +445,7 @@ OLAPStatus TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
return OLAP_SUCCESS;
}
OLAPStatus TabletMeta::to_json(std::string* json_string, json2pb::Pb2JsonOptions& options) {
OLAPStatus TabletMeta::to_json(string* json_string, json2pb::Pb2JsonOptions& options) {
TabletMetaPB tablet_meta_pb;
RETURN_NOT_OK(to_meta_pb(&tablet_meta_pb));
json2pb::ProtoMessageToJson(tablet_meta_pb, json_string, options);
@ -705,7 +721,7 @@ OLAPStatus TabletMeta::set_alter_state(AlterTabletState alter_state) {
}
}
std::string TabletMeta::full_name() const {
string TabletMeta::full_name() const {
std::stringstream ss;
ss << _tablet_id
<< "." << _schema_hash

37
be/src/olap/tablet_meta.h
View File

@ -33,9 +33,6 @@
#include "util/mutex.h"
#include "util/uid_util.h"
using std::string;
using std::vector;
namespace doris {
// Lifecycle states that a Tablet can be in. Legal state transitions for a
@ -107,6 +104,12 @@ public:
uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletMetaSharedPtr* tablet_meta, TabletUid& tablet_uid);
static OLAPStatus create(const TCreateTabletReq& request, const TabletUid& tablet_uid,
uint64_t shard_id, uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletMetaSharedPtr* tablet_meta);
TabletMeta();
TabletMeta(int64_t table_id, int64_t partition_id,
int64_t tablet_id, int32_t schema_hash,
@ -119,13 +122,14 @@ public:
// Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb.
OLAPStatus create_from_file(const std::string& file_path);
OLAPStatus save(const std::string& file_path);
static OLAPStatus save(const string& file_path, TabletMetaPB& tablet_meta_pb);
static OLAPStatus save(const std::string& file_path, TabletMetaPB& tablet_meta_pb);
static OLAPStatus reset_tablet_uid(const std::string& file_path);
static std::string construct_header_file_path(const std::string& schema_hash_path, const int64_t tablet_id);
static std::string construct_header_file_path(const std::string& schema_hash_path,
const int64_t tablet_id);
OLAPStatus save_meta(DataDir* data_dir);
OLAPStatus serialize(string* meta_binary);
OLAPStatus deserialize(const string& meta_binary);
OLAPStatus serialize(std::string* meta_binary);
OLAPStatus deserialize(const std::string& meta_binary);
OLAPStatus init_from_pb(const TabletMetaPB& tablet_meta_pb);
OLAPStatus to_meta_pb(TabletMetaPB* tablet_meta_pb);
@ -157,16 +161,17 @@ public:
inline const TabletSchema& tablet_schema() const;
inline const vector<RowsetMetaSharedPtr>& all_rs_metas() const;
inline const std::vector<RowsetMetaSharedPtr>& all_rs_metas() const;
OLAPStatus add_rs_meta(const RowsetMetaSharedPtr& rs_meta);
RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version) const;
OLAPStatus delete_rs_meta_by_version(const Version& version, vector<RowsetMetaSharedPtr>* deleted_rs_metas);
OLAPStatus modify_rs_metas(const vector<RowsetMetaSharedPtr>& to_add,
const vector<RowsetMetaSharedPtr>& to_delete);
OLAPStatus delete_rs_meta_by_version(const Version& version,
std::vector<RowsetMetaSharedPtr>* deleted_rs_metas);
OLAPStatus modify_rs_metas(const std::vector<RowsetMetaSharedPtr>& to_add,
const std::vector<RowsetMetaSharedPtr>& to_delete);
OLAPStatus revise_rs_metas(const std::vector<RowsetMetaSharedPtr>& rs_metas);
OLAPStatus revise_inc_rs_metas(const std::vector<RowsetMetaSharedPtr>& rs_metas);
inline const vector<RowsetMetaSharedPtr>& all_inc_rs_metas() const;
inline const std::vector<RowsetMetaSharedPtr>& all_inc_rs_metas() const;
OLAPStatus add_inc_rs_meta(const RowsetMetaSharedPtr& rs_meta);
OLAPStatus delete_inc_rs_meta_by_version(const Version& version);
RowsetMetaSharedPtr acquire_inc_rs_meta_by_version(const Version& version) const;
@ -207,8 +212,8 @@ private:
TabletState _tablet_state;
TabletSchema _schema;
vector<RowsetMetaSharedPtr> _rs_metas;
vector<RowsetMetaSharedPtr> _inc_rs_metas;
std::vector<RowsetMetaSharedPtr> _rs_metas;
std::vector<RowsetMetaSharedPtr> _inc_rs_metas;
DelPredicateArray _del_pred_array;
AlterTabletTaskSharedPtr _alter_task;
bool _in_restore_mode = false;
@ -303,11 +308,11 @@ inline const TabletSchema& TabletMeta::tablet_schema() const {
return _schema;
}
inline const vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas() const {
inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas() const {
return _rs_metas;
}
inline const vector<RowsetMetaSharedPtr>& TabletMeta::all_inc_rs_metas() const {
inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_inc_rs_metas() const {
return _inc_rs_metas;
}

11
be/src/service/backend_service.cpp
View File

@ -32,6 +32,7 @@
#include "gen_cpp/PaloInternalService_types.h"
#include "gen_cpp/DorisExternalService_types.h"
#include "gen_cpp/Types_types.h"
#include "gutil/strings/substitute.h"
#include "olap/storage_engine.h"
#include "runtime/external_scan_context_mgr.h"
@ -219,12 +220,12 @@ void BackendService::erase_export_task(TStatus& t_status, const TUniqueId& task_
}
void BackendService::get_tablet_stat(TTabletStatResult& result) {
StorageEngine::instance()->tablet_manager()->get_tablet_stat(result);
StorageEngine::instance()->tablet_manager()->get_tablet_stat(&result);
}
void BackendService::submit_routine_load_task(
TStatus& t_status, const std::vector<TRoutineLoadTask>& tasks) {
for (auto& task : tasks) {
Status st = _exec_env->routine_load_task_executor()->submit_task(task);
if (!st.ok()) {
@ -281,9 +282,9 @@ void BackendService::get_next(TScanBatchResult& result_, const TScanNextBatchPar
LOG(ERROR) << "getNext error: context offset [" << context->offset<<" ]" << " ,client offset [ " << offset << " ]";
// invalid offset
t_status.status_code = TStatusCode::NOT_FOUND;
std::stringstream msg;
msg << "context_id: " << context_id << " send offset: " << offset << "diff with context offset: " << context->offset;
t_status.error_msgs.push_back(msg.str());
t_status.error_msgs.push_back(strings::Substitute(
"context_id=$0, send_offset=$1, context_offset=$2",
context_id, offset, context->offset));
result_.status = t_status;
} else {
// during accessing, should disabled last_access_time

2
be/src/service/backend_service.h
View File

@ -100,7 +100,7 @@ public:
virtual void submit_etl_task(TAgentResult& result,
const TMiniLoadEtlTaskRequest& request) {
VLOG_ROW << "submit_etl_task. request is "
VLOG_RPC << "submit_etl_task. request is "
<< apache::thrift::ThriftDebugString(request).c_str();
_agent_server->submit_etl_task(result, request);
}

1
be/test/olap/tablet_mgr_test.cpp
View File

@ -85,7 +85,6 @@ public:
if (boost::filesystem::exists(_engine_data_path)) {
ASSERT_TRUE(boost::filesystem::remove_all(_engine_data_path));
}
_tablet_mgr.clear();
}
private: