database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Refactor and reorganize the file utils (#2089)

Browse Source
This commit is contained in:
Seaven
2019-11-11 20:25:41 +08:00
committed by ZHAO Chun
parent 068eed8eb0
commit d0316d158d
27 changed files with 581 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
be/src/olap/task/engine_clone_task.cpp
View File

@ -25,6 +25,8 @@
#include "olap/rowset/rowset.h"
#include "olap/rowset/rowset_factory.h"
#include "env/env.h"
using std::set;
using std::stringstream;
@ -194,7 +196,7 @@ OLAPStatus EngineCloneTask::execute() {
}
// clone success, delete .hdr file because tablet meta is stored in rocksdb
string cloned_meta_file = tablet_dir_stream.str() + "/" + std::to_string(_clone_req.tablet_id) + ".hdr";
remove_dir(cloned_meta_file);
FileUtils::remove(cloned_meta_file);
}
// Clean useless dir, if failed, ignore it.
if (status != DORIS_SUCCESS && status != DORIS_CREATE_TABLE_EXIST) {
@ -557,7 +559,7 @@ AgentStatus EngineCloneTask::_clone_copy(
OLAPStatus EngineCloneTask::_convert_to_new_snapshot(const string& clone_dir, int64_t tablet_id) {
OLAPStatus res = OLAP_SUCCESS;
// check clone dir existed
if (!check_dir_existed(clone_dir)) {
if (!FileUtils::check_exist(clone_dir)) {
res = OLAP_ERR_DIR_NOT_EXIST;
LOG(WARNING) << "clone dir not existed when clone. clone_dir=" << clone_dir.c_str();
return res;
@ -580,10 +582,11 @@ OLAPStatus EngineCloneTask::_convert_to_new_snapshot(const string& clone_dir, in
}
set<string> clone_files;
if ((res = dir_walk(clone_dir, NULL, &clone_files)) != OLAP_SUCCESS) {
LOG(WARNING) << "failed to dir walk when clone. [clone_dir=" << clone_dir << "]";
return res;
}
RETURN_WITH_WARN_IF_ERROR(
FileUtils::list_dirs_files(clone_dir, NULL, &clone_files, Env::Default()),
OLAP_ERR_DISK_FAILURE,
"failed to dir walk when clone. clone_dir=" + clone_dir);
try {
olap_header_msg.CopyFrom(file_header.message());
@ -606,8 +609,9 @@ OLAPStatus EngineCloneTask::_convert_to_new_snapshot(const string& clone_dir, in
files_to_delete.push_back(full_file_path);
}
// remove all files
RETURN_NOT_OK(remove_files(files_to_delete));
RETURN_WITH_WARN_IF_ERROR(FileUtils::remove_paths(files_to_delete), OLAP_ERR_IO_ERROR,
"remove paths failed.")
res = TabletMeta::save(cloned_meta_file, tablet_meta_pb);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to save converted tablet meta to dir='" << clone_dir;
@ -631,7 +635,7 @@ OLAPStatus EngineCloneTask::_finish_clone(TabletSharedPtr tablet, const string&
tablet->obtain_header_wrlock();
do {
// check clone dir existed
if (!check_dir_existed(clone_dir)) {
if (!FileUtils::check_exist(clone_dir)) {
res = OLAP_ERR_DIR_NOT_EXIST;
LOG(WARNING) << "clone dir not existed when clone. clone_dir=" << clone_dir.c_str();
break;
@ -646,20 +650,26 @@ OLAPStatus EngineCloneTask::_finish_clone(TabletSharedPtr tablet, const string&
break;
}
// remove the cloned meta file
remove_dir(cloned_tablet_meta_file);
FileUtils::remove(cloned_tablet_meta_file);
// TODO(ygl): convert old format file into rowset
// check all files in /clone and /tablet
set<string> clone_files;
if ((res = dir_walk(clone_dir, NULL, &clone_files)) != OLAP_SUCCESS) {
LOG(WARNING) << "failed to dir walk when clone. [clone_dir=" << clone_dir << "]";
Status ret = FileUtils::list_dirs_files(clone_dir, NULL, &clone_files, Env::Default());
if (!ret.ok()) {
LOG(WARNING) << "failed to dir walk when clone. [clone_dir=" << clone_dir << "]"
<< " error: " << ret.to_string();
res = OLAP_ERR_DISK_FAILURE;
break;
}
set<string> local_files;
string tablet_dir = tablet->tablet_path();
if ((res = dir_walk(tablet_dir, NULL, &local_files)) != OLAP_SUCCESS) {
LOG(WARNING) << "failed to dir walk when clone. [tablet_dir=" << tablet_dir << "]";
ret = FileUtils::list_dirs_files(tablet_dir, NULL, &local_files, Env::Default());
if (!ret.ok()) {
LOG(WARNING) << "failed to dir walk when clone. [tablet_dir=" << tablet_dir << "]"
<< " error: " << ret.to_string();
res = OLAP_ERR_DISK_FAILURE;
break;
}
@ -704,7 +714,7 @@ OLAPStatus EngineCloneTask::_finish_clone(TabletSharedPtr tablet, const string&
// clear linked files if errors happen
if (res != OLAP_SUCCESS) {
remove_files(linked_success_files);
FileUtils::remove_paths(linked_success_files);
}
tablet->release_header_lock();
tablet->release_push_lock();