database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor] A small refactor on class DataDir (#3276)

Browse Source 
main refactor points are:
- Use a single get_absolute_tablet_path function instead of 3
  independent functions
- Remove meaningless return value of register_tablet and deregister_tablet
- Some typo and format
This commit is contained in:
Yingchun Lai
2020-04-09 11:32:22 -05:00
committed by GitHub
parent ce1d5ab9ab
commit f39c8b156d
8 changed files with 142 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
be/src/http/action/restore_tablet_action.cpp
View File

@ -17,28 +17,28 @@
#include "http/action/restore_tablet_action.h"
#include <string>
#include <sstream>
#include <unistd.h>
#include "boost/lexical_cast.hpp"
#include <sstream>
#include <string>
#include "agent/cgroups_mgr.h"
#include "boost/lexical_cast.hpp"
#include "env/env.h"
#include "gutil/strings/substitute.h" // for Substitute
#include "http/http_channel.h"
#include "http/http_headers.h"
#include "http/http_request.h"
#include "http/http_response.h"
#include "http/http_status.h"
#include "util/file_utils.h"
#include "olap/utils.h"
#include "olap/tablet_meta.h"
#include "util/json_util.h"
#include "olap/data_dir.h"
#include "olap/olap_define.h"
#include "olap/storage_engine.h"
#include "olap/data_dir.h"
#include "olap/tablet_meta.h"
#include "olap/utils.h"
#include "runtime/exec_env.h"
#include "gutil/strings/substitute.h" // for Substitute
#include "util/file_utils.h"
#include "util/json_util.h"
using boost::filesystem::path;
@ -47,10 +47,9 @@ namespace doris {
const std::string TABLET_ID = "tablet_id";
const std::string SCHEMA_HASH = "schema_hash";
RestoreTabletAction::RestoreTabletAction(ExecEnv* exec_env) : _exec_env(exec_env) {
}
RestoreTabletAction::RestoreTabletAction(ExecEnv* exec_env) : _exec_env(exec_env) {}
void RestoreTabletAction::handle(HttpRequest *req) {
void RestoreTabletAction::handle(HttpRequest* req) {
LOG(INFO) << "accept one request " << req->debug_string();
// add tid to cgroup in order to limit read bandwidth
CgroupsMgr::apply_system_cgroup();
@ -64,20 +63,18 @@ void RestoreTabletAction::handle(HttpRequest *req) {
}
}
Status RestoreTabletAction::_handle(HttpRequest *req) {
Status RestoreTabletAction::_handle(HttpRequest* req) {
// Get tablet id
const std::string& tablet_id_str = req->param(TABLET_ID);
if (tablet_id_str.empty()) {
std::string error_msg = std::string(
"parameter " + TABLET_ID + " not specified in url.");
std::string error_msg = std::string("parameter " + TABLET_ID + " not specified in url.");
return Status::InternalError(error_msg);
}
// Get schema hash
const std::string& schema_hash_str = req->param(SCHEMA_HASH);
if (schema_hash_str.empty()) {
std::string error_msg = std::string(
"parameter " + SCHEMA_HASH + " not specified in url.");
std::string error_msg = std::string("parameter " + SCHEMA_HASH + " not specified in url.");
return Status::InternalError(error_msg);
}
@ -97,33 +94,36 @@ Status RestoreTabletAction::_handle(HttpRequest *req) {
// check tablet_id + schema_hash already is restoring
std::lock_guard<std::mutex> l(_tablet_restore_lock);
if (_tablet_path_map.find(key) != _tablet_path_map.end()) {
LOG(INFO) << "tablet_id:" << tablet_id << " schema_hash:" << schema_hash << " is restoring.";
LOG(INFO) << "tablet_id:" << tablet_id << " schema_hash:" << schema_hash
<< " is restoring.";
return Status::InternalError("tablet is already restoring");
} else {
// set key in map and initialize value as ""
_tablet_path_map[key] = "";
LOG(INFO) << "start to restore tablet_id:" << tablet_id << " schema_hash:" << schema_hash;
LOG(INFO) << "start to restore tablet_id:" << tablet_id
<< " schema_hash:" << schema_hash;
}
}
Status status = _restore(key, tablet_id, schema_hash);
_clear_key(key);
LOG(INFO) << "deal with restore tablet request finished! tablet id: " << tablet_id << "-" << schema_hash;
LOG(INFO) << "deal with restore tablet request finished! tablet id: " << tablet_id << "-"
<< schema_hash;
return status;
}
Status RestoreTabletAction::_reload_tablet(
const std::string& key, const std::string& shard_path, int64_t tablet_id, int32_t schema_hash) {
Status RestoreTabletAction::_reload_tablet(const std::string& key, const std::string& shard_path,
int64_t tablet_id, int32_t schema_hash) {
TCloneReq clone_req;
clone_req.__set_tablet_id(tablet_id);
clone_req.__set_schema_hash(schema_hash);
OLAPStatus res = OLAPStatus::OLAP_SUCCESS;
res = _exec_env->storage_engine()->load_header(shard_path, clone_req);
if (res != OLAPStatus::OLAP_SUCCESS) {
LOG(WARNING) << "load header failed. status: " << res
<< ", signature: " << tablet_id;
LOG(WARNING) << "load header failed. status: " << res << ", signature: " << tablet_id;
// remove tablet data path in data path
// path: /roo_path/data/shard/tablet_id
std::string tablet_path = strings::Substitute("$0/$1/$2", shard_path, tablet_id, schema_hash);
std::string tablet_path =
strings::Substitute("$0/$1/$2", shard_path, tablet_id, schema_hash);
LOG(INFO) << "remove tablet_path:" << tablet_path;
Status s = FileUtils::remove_all(tablet_path);
if (!s.ok()) {
@ -137,25 +137,26 @@ Status RestoreTabletAction::_reload_tablet(
std::lock_guard<std::mutex> l(_tablet_restore_lock);
trash_tablet_schema_hash_dir = _tablet_path_map[key];
}
LOG(INFO) << "load header success. status: " << res
<< ", signature: " << tablet_id << ", from trash path:" << trash_tablet_schema_hash_dir
LOG(INFO) << "load header success. status: " << res << ", signature: " << tablet_id
<< ", from trash path:" << trash_tablet_schema_hash_dir
<< " to shard path:" << shard_path;
return Status::OK();
}
}
Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id, int32_t schema_hash) {
Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id,
int32_t schema_hash) {
// get latest tablet path in trash
std::string latest_tablet_path;
bool ret = _get_latest_tablet_path_from_trash(tablet_id, schema_hash, &latest_tablet_path);
if (!ret) {
LOG(WARNING) << "can not find tablet:" << tablet_id
<< ", schema hash:" << schema_hash;
LOG(WARNING) << "can not find tablet:" << tablet_id << ", schema hash:" << schema_hash;
return Status::InternalError("can find tablet path in trash");
}
LOG(INFO) << "tablet path in trash:" << latest_tablet_path;
std::string original_header_path = latest_tablet_path + "/" + std::to_string(tablet_id) +".hdr";
std::string original_header_path =
latest_tablet_path + "/" + std::to_string(tablet_id) + ".hdr";
TabletMeta tablet_meta;
OLAPStatus load_status = tablet_meta.create_from_file(original_header_path);
if (load_status != OLAP_SUCCESS) {
@ -169,9 +170,11 @@ Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id,
_tablet_path_map[key] = latest_tablet_path;
}
std::string root_path = DataDir::get_root_path_from_schema_hash_path_in_trash(latest_tablet_path);
std::string root_path =
DataDir::get_root_path_from_schema_hash_path_in_trash(latest_tablet_path);
DataDir* store = StorageEngine::instance()->get_store(root_path);
std::string restore_schema_hash_path = store->get_absolute_tablet_path(&tablet_meta, true);
std::string restore_schema_hash_path = store->get_absolute_tablet_path(
tablet_meta.shard_id(), tablet_meta.tablet_id(), tablet_meta.schema_hash());
Status s = FileUtils::create_dir(restore_schema_hash_path);
if (!s.ok()) {
LOG(WARNING) << "create tablet path failed:" << restore_schema_hash_path;
@ -183,12 +186,13 @@ Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id,
RETURN_IF_ERROR(FileUtils::remove_all(restore_schema_hash_path));
return s;
}
std::string restore_shard_path = store->get_absolute_shard_path(std::to_string(tablet_meta.shard_id()));
std::string restore_shard_path = store->get_absolute_shard_path(tablet_meta.shard_id());
Status status = _reload_tablet(key, restore_shard_path, tablet_id, schema_hash);
return status;
}
Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src, const std::string& dst) {
Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src,
const std::string& dst) {
std::vector<std::string> files;
RETURN_IF_ERROR(FileUtils::list_files(Env::Default(), src, &files));
for (auto& file : files) {
@ -200,8 +204,8 @@ Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src,
} else {
int link_ret = link(from.c_str(), to.c_str());
if (link_ret != 0) {
LOG(WARNING) << "link from:" << from
<< " to:" << to << " failed, link ret:" << link_ret;
LOG(WARNING) << "link from:" << from << " to:" << to
<< " failed, link ret:" << link_ret;
return Status::InternalError("create link path failed");
}
}
@ -209,8 +213,8 @@ Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src,
return Status::OK();
}
bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
int64_t tablet_id, int32_t schema_hash, std::string* path) {
bool RestoreTabletAction::_get_latest_tablet_path_from_trash(int64_t tablet_id, int32_t schema_hash,
std::string* path) {
std::vector<std::string> tablet_paths;
std::vector<DataDir*> stores = StorageEngine::instance()->get_stores();
for (auto& store : stores) {
@ -229,8 +233,7 @@ bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
}
}
if (schema_hash_paths.size() == 0) {
LOG(WARNING) << "can not find tablet_id:" << tablet_id
<< ", schema_hash:" << schema_hash;;
LOG(WARNING) << "can not find tablet_id:" << tablet_id << ", schema_hash:" << schema_hash;
return false;
} else if (schema_hash_paths.size() == 1) {
*path = schema_hash_paths[0];
@ -240,9 +243,10 @@ bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
uint64_t max_timestamp = 0;
uint64_t max_counter = 0;
*path = schema_hash_paths[start_index];
if (!_get_timestamp_and_count_from_schema_hash_path(
schema_hash_paths[start_index], &max_timestamp, &max_counter)) {
LOG(WARNING) << "schema hash paths are invalid, path:" << schema_hash_paths[start_index];
if (!_get_timestamp_and_count_from_schema_hash_path(schema_hash_paths[start_index],
&max_timestamp, &max_counter)) {
LOG(WARNING) << "schema hash paths are invalid, path:"
<< schema_hash_paths[start_index];
return false;
}
// find latest path
@ -250,7 +254,7 @@ bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
uint64_t current_timestamp = 0;
uint64_t current_counter = 0;
if (!_get_timestamp_and_count_from_schema_hash_path(
schema_hash_paths[i], &current_timestamp, &current_counter)) {
schema_hash_paths[i], &current_timestamp, &current_counter)) {
LOG(WARNING) << "schema hash path:" << schema_hash_paths[i] << " is invalid";
continue;
}

84
be/src/olap/data_dir.cpp
View File

@ -35,6 +35,7 @@
#include <sstream>
#include "env/env.h"
#include "gutil/strings/substitute.h"
#include "olap/file_helper.h"
#include "olap/olap_define.h"
#include "olap/olap_snapshot_converter.h"
@ -397,68 +398,35 @@ OLAPStatus DataDir::get_shard(uint64_t* shard) {
return OLAP_SUCCESS;
}
OLAPStatus DataDir::register_tablet(Tablet* tablet) {
std::lock_guard<std::mutex> l(_mutex);
void DataDir::register_tablet(Tablet* tablet) {
TabletInfo tablet_info(tablet->tablet_id(), tablet->schema_hash(), tablet->tablet_uid());
_tablet_set.insert(tablet_info);
return OLAP_SUCCESS;
std::lock_guard<std::mutex> l(_mutex);
_tablet_set.emplace(std::move(tablet_info));
}
OLAPStatus DataDir::deregister_tablet(Tablet* tablet) {
std::lock_guard<std::mutex> l(_mutex);
void DataDir::deregister_tablet(Tablet* tablet) {
TabletInfo tablet_info(tablet->tablet_id(), tablet->schema_hash(), tablet->tablet_uid());
std::lock_guard<std::mutex> l(_mutex);
_tablet_set.erase(tablet_info);
return OLAP_SUCCESS;
}
void DataDir::clear_tablets(std::vector<TabletInfo>* tablet_infos) {
for (auto& tablet : _tablet_set) {
tablet_infos->push_back(tablet);
}
std::lock_guard<std::mutex> l(_mutex);
tablet_infos->insert(tablet_infos->end(), _tablet_set.begin(), _tablet_set.end());
_tablet_set.clear();
}
std::string DataDir::get_absolute_shard_path(const std::string& shard_string) {
return _path + DATA_PREFIX + "/" + shard_string;
std::string DataDir::get_absolute_shard_path(int64_t shard_id) {
return strings::Substitute("$0$1/$2", _path, DATA_PREFIX, shard_id);
}
std::string DataDir::get_absolute_tablet_path(TabletMeta* tablet_meta, bool with_schema_hash) {
if (with_schema_hash) {
return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
std::to_string(tablet_meta->tablet_id()) + "/" +
std::to_string(tablet_meta->schema_hash());
} else {
return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
std::to_string(tablet_meta->tablet_id());
}
}
std::string DataDir::get_absolute_tablet_path(TabletMetaPB* tablet_meta, bool with_schema_hash) {
if (with_schema_hash) {
return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
std::to_string(tablet_meta->tablet_id()) + "/" +
std::to_string(tablet_meta->schema_hash());
} else {
return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
std::to_string(tablet_meta->tablet_id());
}
}
std::string DataDir::get_absolute_tablet_path(OLAPHeaderMessage& olap_header_msg,
bool with_schema_hash) {
if (with_schema_hash) {
return _path + DATA_PREFIX + "/" + std::to_string(olap_header_msg.shard()) + "/" +
std::to_string(olap_header_msg.tablet_id()) + "/" +
std::to_string(olap_header_msg.schema_hash());
} else {
return _path + DATA_PREFIX + "/" + std::to_string(olap_header_msg.shard()) + "/" +
std::to_string(olap_header_msg.tablet_id());
}
std::string DataDir::get_absolute_tablet_path(int64_t shard_id, int64_t tablet_id,
int32_t schema_hash) {
return strings::Substitute("$0/$1/$2", get_absolute_shard_path(shard_id), tablet_id,
schema_hash);
}
void DataDir::find_tablet_in_trash(int64_t tablet_id, std::vector<std::string>* paths) {
@ -540,7 +508,9 @@ OLAPStatus DataDir::_convert_old_tablet() {
<< tablet_id << "." << schema_hash;
return false;
}
string old_data_path_prefix = get_absolute_tablet_path(olap_header_msg, true);
string old_data_path_prefix =
get_absolute_tablet_path(olap_header_msg.shard_id(), olap_header_msg.tablet_id(),
olap_header_msg.schema_hash());
OLAPStatus status = converter.to_new_snapshot(olap_header_msg, old_data_path_prefix,
old_data_path_prefix, &tablet_meta_pb,
&pending_rowsets, true);
@ -617,7 +587,9 @@ OLAPStatus DataDir::remove_old_meta_and_files() {
TabletSchema tablet_schema;
tablet_schema.init_from_pb(tablet_meta_pb.schema());
string data_path_prefix = get_absolute_tablet_path(&tablet_meta_pb, true);
string data_path_prefix =
get_absolute_tablet_path(tablet_meta_pb.shard_id(), tablet_meta_pb.tablet_id(),
tablet_meta_pb.schema_hash());
// convert visible pdelta file to rowsets and remove old files
for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
@ -697,8 +669,8 @@ OLAPStatus DataDir::load() {
// if one rowset load failed, then the total data dir will not be loaded
std::vector<RowsetMetaSharedPtr> dir_rowset_metas;
LOG(INFO) << "begin loading rowset from meta";
auto load_rowset_func = [this, &dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,
const std::string& meta_str) -> bool {
auto load_rowset_func = [&dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,
const std::string& meta_str) -> bool {
RowsetMetaSharedPtr rowset_meta(new AlphaRowsetMeta());
bool parsed = rowset_meta->init(meta_str);
if (!parsed) {
@ -741,7 +713,7 @@ OLAPStatus DataDir::load() {
LOG(INFO) << "load rowset from meta finished, data dir: " << _path;
}
// tranverse rowset
// traverse rowset
// 1. add committed rowset to txn map
// 2. add visible rowset to tablet
// ignore any errors when load tablet or rowset, because fe will repair them after report
@ -843,7 +815,7 @@ void DataDir::perform_path_gc_by_rowsetid() {
// gc thread should get tablet include deleted tablet
// or it will delete rowset file before tablet is garbage collected
RowsetId rowset_id;
bool is_rowset_file = _tablet_manager->get_rowset_id_from_path(path, &rowset_id);
bool is_rowset_file = TabletManager::get_rowset_id_from_path(path, &rowset_id);
if (is_rowset_file) {
TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id, schema_hash);
if (tablet != nullptr) {
@ -863,7 +835,7 @@ void DataDir::perform_path_gc_by_rowsetid() {
void DataDir::perform_path_scan() {
{
std::unique_lock<std::mutex> lck(_check_path_mutex);
if (_all_check_paths.size() > 0) {
if (!_all_check_paths.empty()) {
LOG(INFO) << "_all_check_paths is not empty when path scan.";
return;
}

41
be/src/olap/data_dir.h
View File

@ -17,11 +17,11 @@
#pragma once
#include <condition_variable>
#include <cstdint>
#include <mutex>
#include <set>
#include <string>
#include <mutex>
#include <condition_variable>
#include "common/status.h"
#include "gen_cpp/Types_types.h"
@ -37,15 +37,13 @@ class TabletManager;
class TabletMeta;
class TxnManager;
// A DataDir used to manange data in same path.
// A DataDir used to manage data in same path.
// Now, After DataDir was created, it will never be deleted for easy implementation.
class DataDir {
public:
DataDir(const std::string& path,
int64_t capacity_bytes = -1,
DataDir(const std::string& path, int64_t capacity_bytes = -1,
TStorageMedium::type storage_medium = TStorageMedium::HDD,
TabletManager* tablet_manager = nullptr,
TxnManager* txn_manager = nullptr);
TabletManager* tablet_manager = nullptr, TxnManager* txn_manager = nullptr);
~DataDir();
Status init();
@ -78,27 +76,21 @@ public:
OlapMeta* get_meta() { return _meta; }
bool is_ssd_disk() const {
return _storage_medium == TStorageMedium::SSD;
}
bool is_ssd_disk() const { return _storage_medium == TStorageMedium::SSD; }
TStorageMedium::type storage_medium() const { return _storage_medium; }
OLAPStatus register_tablet(Tablet* tablet);
OLAPStatus deregister_tablet(Tablet* tablet);
void register_tablet(Tablet* tablet);
void deregister_tablet(Tablet* tablet);
void clear_tablets(std::vector<TabletInfo>* tablet_infos);
std::string get_absolute_tablet_path(TabletMeta* tablet_meta, bool with_schema_hash);
std::string get_absolute_tablet_path(OLAPHeaderMessage& olap_header_msg, bool with_schema_hash);
std::string get_absolute_tablet_path(TabletMetaPB* tablet_meta, bool with_schema_hash);
std::string get_absolute_shard_path(const std::string& shard_string);
std::string get_absolute_shard_path(int64_t shard_id);
std::string get_absolute_tablet_path(int64_t shard_id, int64_t tablet_id, int32_t schema_hash);
void find_tablet_in_trash(int64_t tablet_id, std::vector<std::string>* paths);
static std::string get_root_path_from_schema_hash_path_in_trash(const std::string& schema_hash_dir_in_trash);
static std::string get_root_path_from_schema_hash_path_in_trash(
const std::string& schema_hash_dir_in_trash);
// load data from meta and data files
OLAPStatus load();
@ -122,8 +114,8 @@ public:
// check if the capacity reach the limit after adding the incoming data
// return true if limit reached, otherwise, return false.
// TODO(cmy): for now we can not precisely calculate the capacity Doris used,
// so in order to avoid running out of disk capacity, we currenty use the actual
// disk avaiable capacity and total capacity to do the calculation.
// so in order to avoid running out of disk capacity, we currently use the actual
// disk available capacity and total capacity to do the calculation.
// So that the capacity Doris actually used may exceeds the user specified capacity.
bool reach_capacity_limit(int64_t incoming_data_size);
@ -186,15 +178,16 @@ private:
static const uint32_t MAX_SHARD_NUM = 1024;
char* _test_file_read_buf;
char* _test_file_write_buf;
OlapMeta* _meta = nullptr;
RowsetIdGenerator* _id_generator = nullptr;
std::set<std::string> _all_check_paths;
std::mutex _check_path_mutex;
std::condition_variable _cv;
std::set<std::string> _all_check_paths;
std::set<std::string> _pending_path_ids;
RWMutex _pending_path_mutex;
std::set<std::string> _pending_path_ids;
// used in convert process
bool _convert_old_data_success;

6
be/src/olap/olap_snapshot_converter.cpp
View File

@ -89,7 +89,7 @@ OLAPStatus OlapSnapshotConverter::to_olap_header(const TabletMetaPB& tablet_meta
olap_header->set_schema_hash(tablet_meta_pb.schema_hash());
}
if (tablet_meta_pb.has_shard_id()) {
olap_header->set_shard(tablet_meta_pb.shard_id());
olap_header->set_shard_id(tablet_meta_pb.shard_id());
}
return OLAP_SUCCESS;
}
@ -102,8 +102,8 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola
if (olap_header.has_schema_hash()) {
tablet_meta_pb->set_schema_hash(olap_header.schema_hash());
}
if (olap_header.has_shard()) {
tablet_meta_pb->set_shard_id(olap_header.shard());
if (olap_header.has_shard_id()) {
tablet_meta_pb->set_shard_id(olap_header.shard_id());
}
tablet_meta_pb->set_creation_time(olap_header.creation_time());
tablet_meta_pb->set_cumulative_layer_point(-1);

12
be/src/olap/tablet.h
View File

@ -59,8 +59,8 @@ public:
bool is_used();
inline DataDir* data_dir() const;
OLAPStatus register_tablet_into_dir();
OLAPStatus deregister_tablet_from_dir();
void register_tablet_into_dir();
void deregister_tablet_from_dir();
std::string tablet_path() const;
@ -324,12 +324,12 @@ inline DataDir* Tablet::data_dir() const {
return _data_dir;
}
inline OLAPStatus Tablet::register_tablet_into_dir() {
return _data_dir->register_tablet(this);
inline void Tablet::register_tablet_into_dir() {
_data_dir->register_tablet(this);
}
inline OLAPStatus Tablet::deregister_tablet_from_dir() {
return _data_dir->deregister_tablet(this);
inline void Tablet::deregister_tablet_from_dir() {
_data_dir->deregister_tablet(this);
}
inline string Tablet::tablet_path() const {

8
be/src/olap/tablet_manager.cpp
View File

@ -184,9 +184,7 @@ OLAPStatus TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id, Schem
// Register tablet into DataDir, so that we can manage tablet from
// the perspective of root path.
// Example: unregister all tables when a bad disk found.
RETURN_NOT_OK_LOG(tablet->register_tablet_into_dir(), Substitute(
"fail to register tablet into StorageEngine. data_dir=$0",
tablet->data_dir()->path()));
tablet->register_tablet_into_dir();
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
tablet_map[tablet_id].table_arr.push_back(tablet);
tablet_map[tablet_id].table_arr.sort(_cmp_tablet_by_create_time);
@ -1337,9 +1335,7 @@ OLAPStatus TabletManager::_drop_tablet_directly_unlocked(
}
}
RETURN_NOT_OK_LOG(dropped_tablet->deregister_tablet_from_dir(), Substitute(
"fail to unregister from root path. tablet=$0, schema_hash=$1",
tablet_id, schema_hash));
dropped_tablet->deregister_tablet_from_dir();
return OLAP_SUCCESS;
}

72
be/test/olap/olap_snapshot_converter_test.cpp
View File

@ -15,24 +15,25 @@
// specific language governing permissions and limitations
// under the License.
#include <iostream>
#include <string>
#include <sstream>
#include <fstream>
#include "olap/olap_snapshot_converter.h"
#include "gtest/gtest.h"
#include <boost/algorithm/string.hpp>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include "boost/filesystem.hpp"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "json2pb/json_to_pb.h"
#include "olap/lru_cache.h"
#include "olap/olap_meta.h"
#include "olap/olap_snapshot_converter.h"
#include "olap/rowset/rowset_meta_manager.h"
#include "olap/rowset/alpha_rowset.h"
#include "olap/rowset/alpha_rowset_meta.h"
#include "olap/rowset/rowset_meta_manager.h"
#include "olap/storage_engine.h"
#include "olap/txn_manager.h"
#include <boost/algorithm/string.hpp>
#include "boost/filesystem.hpp"
#include "json2pb/json_to_pb.h"
#include "util/file_utils.h"
#ifndef BE_TEST
@ -77,16 +78,14 @@ public:
copy_dir(test_engine_data_path, tmp_data_path);
_tablet_id = 15007;
_schema_hash = 368169781;
_tablet_data_path = tmp_data_path
+ "/" + std::to_string(0)
+ "/" + std::to_string(_tablet_id)
+ "/" + std::to_string(_schema_hash);
_tablet_data_path = tmp_data_path + "/" + std::to_string(0) + "/" +
std::to_string(_tablet_id) + "/" + std::to_string(_schema_hash);
if (boost::filesystem::exists(_meta_path)) {
boost::filesystem::remove_all(_meta_path);
}
ASSERT_TRUE(boost::filesystem::create_directory(_meta_path));
ASSERT_TRUE(boost::filesystem::exists(_meta_path));
_meta = new(std::nothrow) OlapMeta(_meta_path);
_meta = new (std::nothrow) OlapMeta(_meta_path);
ASSERT_NE(nullptr, _meta);
OLAPStatus st = _meta->init();
ASSERT_TRUE(st == OLAP_SUCCESS);
@ -131,12 +130,13 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
TabletMetaPB tablet_meta_pb;
vector<RowsetMetaPB> pending_rowsets;
OLAPStatus status = converter.to_new_snapshot(header_msg, _tablet_data_path, _tablet_data_path,
&tablet_meta_pb, &pending_rowsets, true);
&tablet_meta_pb, &pending_rowsets, true);
ASSERT_TRUE(status == OLAP_SUCCESS);
TabletSchema tablet_schema;
tablet_schema.init_from_pb(tablet_meta_pb.schema());
string data_path_prefix = _data_dir->get_absolute_tablet_path(&tablet_meta_pb, true);
string data_path_prefix = _data_dir->get_absolute_tablet_path(
tablet_meta_pb.shard_id(), tablet_meta_pb.tablet_id(), tablet_meta_pb.schema_hash());
// check converted new tabletmeta pb and its files
// check visible delta
ASSERT_TRUE(tablet_meta_pb.rs_metas().size() == header_msg.delta().size());
@ -145,8 +145,8 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
int64_t end_version = pdelta.end_version();
bool found = false;
for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
if (visible_rowset.start_version() == start_version
&& visible_rowset.end_version() == end_version) {
if (visible_rowset.start_version() == start_version &&
visible_rowset.end_version() == end_version) {
found = true;
}
}
@ -169,9 +169,9 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
int64_t version_hash = pdelta.version_hash();
bool found = false;
for (auto& inc_rowset : tablet_meta_pb.inc_rs_metas()) {
if (inc_rowset.start_version() == start_version
&& inc_rowset.end_version() == end_version
&& inc_rowset.version_hash() == version_hash) {
if (inc_rowset.start_version() == start_version &&
inc_rowset.end_version() == end_version &&
inc_rowset.version_hash() == version_hash) {
found = true;
}
}
@ -184,7 +184,7 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
ASSERT_TRUE(rowset.init() == OLAP_SUCCESS);
ASSERT_TRUE(rowset.load() == OLAP_SUCCESS);
AlphaRowset tmp_rowset(&tablet_schema, data_path_prefix + "/incremental_delta",
alpha_rowset_meta);
alpha_rowset_meta);
ASSERT_TRUE(tmp_rowset.init() == OLAP_SUCCESS);
std::vector<std::string> old_files;
tmp_rowset.remove_old_files(&old_files);
@ -196,10 +196,10 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
int64_t transaction_id = pdelta.transaction_id();
bool found = false;
for (auto& pending_rowset : pending_rowsets) {
if (pending_rowset.partition_id() == partition_id
&& pending_rowset.txn_id() == transaction_id
&& pending_rowset.tablet_uid().hi() == tablet_meta_pb.tablet_uid().hi()
&& pending_rowset.tablet_uid().lo() == tablet_meta_pb.tablet_uid().lo()) {
if (pending_rowset.partition_id() == partition_id &&
pending_rowset.txn_id() == transaction_id &&
pending_rowset.tablet_uid().hi() == tablet_meta_pb.tablet_uid().hi() &&
pending_rowset.tablet_uid().lo() == tablet_meta_pb.tablet_uid().lo()) {
found = true;
}
}
@ -217,14 +217,14 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
// old files are removed, then convert new snapshot to old snapshot
OLAPHeaderMessage old_header_msg;
status = converter.to_old_snapshot(tablet_meta_pb, _tablet_data_path,
_tablet_data_path, &old_header_msg);
status = converter.to_old_snapshot(tablet_meta_pb, _tablet_data_path, _tablet_data_path,
&old_header_msg);
ASSERT_TRUE(status == OLAP_SUCCESS);
for (auto& pdelta : header_msg.delta()) {
bool found = false;
for (auto& converted_pdelta : old_header_msg.delta()) {
if (converted_pdelta.start_version() == pdelta.start_version()
&& converted_pdelta.end_version() == pdelta.end_version()) {
if (converted_pdelta.start_version() == pdelta.start_version() &&
converted_pdelta.end_version() == pdelta.end_version()) {
found = true;
}
}
@ -233,9 +233,9 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
for (auto& pdelta : header_msg.incremental_delta()) {
bool found = false;
for (auto& converted_pdelta : old_header_msg.incremental_delta()) {
if (converted_pdelta.start_version() == pdelta.start_version()
&& converted_pdelta.end_version() == pdelta.end_version()
&& converted_pdelta.version_hash() == pdelta.version_hash()) {
if (converted_pdelta.start_version() == pdelta.start_version() &&
converted_pdelta.end_version() == pdelta.end_version() &&
converted_pdelta.version_hash() == pdelta.version_hash()) {
found = true;
}
}
@ -243,9 +243,9 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
}
}
} // namespace doris
} // namespace doris
int main(int argc, char **argv) {
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

2
gensrc/proto/olap_file.proto
View File

@ -211,7 +211,7 @@ message OLAPHeaderMessage {
optional bool in_restore_mode = 19 [default = false]; // TabletMetaPB.is_restore_mode
optional int64 tablet_id = 20; // TabletMetaPB.tablet_id
optional int32 schema_hash = 21; // TabletMetaPB.schema_hash? int32 vs int64
optional uint64 shard = 22; // TabletMetaPB.shard_id? int64 vs int32
optional uint64 shard_id = 22; // TabletMetaPB.shard_id? int64 vs int32
}
enum AlterTabletState {