database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Support setting properties for storage_root_path (#2235)

Browse Source 
We can specify the properties of storage_root_path by setting ':', seperate by ','
e.g.
storage_root_path = /home/disk1/palo,medium:ssd,capacity:50
This commit is contained in:
HuangWei
2019-11-22 18:12:26 +08:00
committed by ZHAO Chun
parent 0e84a88c1a
commit fda46654a2
10 changed files with 290 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
be/src/olap/data_dir.cpp
View File

@ -56,20 +56,22 @@ static const char* const kMtabPath = "/etc/mtab";
static const char* const kTestFilePath = "/.testfile";
DataDir::DataDir(const std::string& path, int64_t capacity_bytes,
TabletManager* tablet_manager, TxnManager* txn_manager)
: _path(path),
_capacity_bytes(capacity_bytes),
_available_bytes(0),
_disk_capacity_bytes(0),
_is_used(false),
_tablet_manager(tablet_manager),
_txn_manager(txn_manager),
_cluster_id(-1),
_to_be_deleted(false),
_current_shard(0),
_test_file_read_buf(nullptr),
_test_file_write_buf(nullptr),
_meta(nullptr) {
TStorageMedium::type storage_medium,
TabletManager* tablet_manager, TxnManager* txn_manager)
: _path(path),
_capacity_bytes(capacity_bytes),
_available_bytes(0),
_disk_capacity_bytes(0),
_storage_medium(storage_medium),
_is_used(false),
_tablet_manager(tablet_manager),
_txn_manager(txn_manager),
_cluster_id(-1),
_to_be_deleted(false),
_current_shard(0),
_test_file_read_buf(nullptr),
_test_file_write_buf(nullptr),
_meta(nullptr) {
}
DataDir::~DataDir() {
@ -105,7 +107,7 @@ Status DataDir::init() {
RETURN_IF_ERROR(update_capacity());
RETURN_IF_ERROR(_init_cluster_id());
RETURN_IF_ERROR(_init_extension_and_capacity());
RETURN_IF_ERROR(_init_capacity());
RETURN_IF_ERROR(_init_file_system());
RETURN_IF_ERROR(_init_meta());
@ -175,22 +177,8 @@ Status DataDir::_read_cluster_id(const std::string& path, int32_t* cluster_id) {
return Status::OK();
}
Status DataDir::_init_extension_and_capacity() {
Status DataDir::_init_capacity() {
boost::filesystem::path boost_path = _path;
std::string extension = boost::filesystem::canonical(boost_path).extension().string();
if (extension != "") {
if (boost::iequals(extension, ".ssd")) {
_storage_medium = TStorageMedium::SSD;
} else if (boost::iequals(extension, ".hdd")) {
_storage_medium = TStorageMedium::HDD;
} else {
LOG(WARNING) << "store path has wrong extension. path=" << _path;
return Status::InternalError("invalid sotre path: invalid extension");
}
} else {
_storage_medium = TStorageMedium::HDD;
}
int64_t disk_capacity = boost::filesystem::space(boost_path).capacity;
if (_capacity_bytes == -1) {
_capacity_bytes = disk_capacity;

3
be/src/olap/data_dir.h
View File

@ -37,6 +37,7 @@ class DataDir {
public:
DataDir(const std::string& path,
int64_t capacity_bytes = -1,
TStorageMedium::type storage_medium = TStorageMedium::HDD,
TabletManager* tablet_manager = nullptr,
TxnManager* txn_manager = nullptr);
~DataDir();
@ -129,7 +130,7 @@ public:
private:
std::string _cluster_id_path() const { return _path + CLUSTER_ID_PREFIX; }
Status _init_cluster_id();
Status _init_extension_and_capacity();
Status _init_capacity();
Status _init_file_system();
Status _init_meta();

114
be/src/olap/options.cpp
View File

@ -17,8 +17,8 @@
#include "olap/options.h"
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include "common/logging.h"
@ -29,36 +29,101 @@ namespace doris {
// compatible with old multi path configuration:
// /path1,2014;/path2,2048
OLAPStatus parse_conf_store_paths(
const std::string& config_path,
std::vector<StorePath>* paths) {
OLAPStatus parse_root_path(const std::string& root_path, StorePath* path) {
try {
std::vector<std::string> item_vec;
boost::split(item_vec, config_path, boost::is_any_of(";"), boost::token_compress_on);
for (auto& item : item_vec) {
std::vector<std::string> tmp_vec;
boost::split(tmp_vec, item, boost::is_any_of(","), boost::token_compress_on);
std::vector<std::string> tmp_vec;
boost::split(tmp_vec, root_path, boost::is_any_of(","),
boost::token_compress_on);
// parse root path name
boost::trim(tmp_vec[0]);
tmp_vec[0].erase(tmp_vec[0].find_last_not_of("/") + 1);
if (tmp_vec[0].empty() || tmp_vec[0][0] != '/') {
LOG(WARNING) << "invalid store path. path=" << tmp_vec[0];
// parse root path name
boost::trim(tmp_vec[0]);
tmp_vec[0].erase(tmp_vec[0].find_last_not_of("/") + 1);
if (tmp_vec[0].empty() || tmp_vec[0][0] != '/') {
LOG(WARNING) << "invalid store path. path=" << tmp_vec[0];
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
path->path = tmp_vec[0];
// parse root path capacity and storage medium
std::string capacity_str, medium_str;
boost::filesystem::path boost_path = tmp_vec[0];
std::string extension =
boost::filesystem::canonical(boost_path).extension().string();
if (!extension.empty()) {
medium_str = extension.substr(1);
}
for (int i = 1; i < tmp_vec.size(); i++) {
// <property>:<value> or <value>
std::string property, value;
std::size_t found = tmp_vec[i].find(':');
if (found != std::string::npos) {
property = boost::trim_copy(tmp_vec[i].substr(0, found));
value = boost::trim_copy(tmp_vec[i].substr(found + 1));
} else {
// <value> only supports setting capacity
property = "capacity";
value = boost::trim_copy(tmp_vec[i]);
}
if (boost::iequals(property, "capacity")) {
capacity_str = value;
} else if (boost::iequals(property, "medium")) {
// property 'medium' has a higher priority than the extension of
// path, so it can override medium_str
medium_str = value;
} else {
LOG(WARNING) << "invalid property of store path, " << property;
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
}
// parse root path capacity
int64_t capacity_bytes = -1;
if (tmp_vec.size() > 1) {
if (!valid_signed_number<int64_t>(tmp_vec[1])
|| strtol(tmp_vec[1].c_str(), NULL, 10) < 0) {
LOG(WARNING) << "invalid capacity of store path, capacity=" << tmp_vec[1];
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
capacity_bytes = strtol(tmp_vec[1].c_str(), NULL, 10) * GB_EXCHANGE_BYTE;
path->capacity_bytes = -1;
if (!capacity_str.empty()) {
if (!valid_signed_number<int64_t>(capacity_str) ||
strtol(capacity_str.c_str(), NULL, 10) < 0) {
LOG(WARNING) << "invalid capacity of store path, capacity="
<< capacity_str;
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
path->capacity_bytes =
strtol(capacity_str.c_str(), NULL, 10) * GB_EXCHANGE_BYTE;
}
paths->emplace_back(tmp_vec[0], capacity_bytes);
path->storage_medium = TStorageMedium::HDD;
if (!medium_str.empty()) {
if (boost::iequals(medium_str, "ssd")) {
path->storage_medium = TStorageMedium::SSD;
} else if (boost::iequals(medium_str, "hdd")) {
path->storage_medium = TStorageMedium::HDD;
} else {
LOG(WARNING) << "invalid storage medium. medium=" << medium_str;
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
}
} catch (...) {
LOG(WARNING) << "invalid store path. path=" << root_path;
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
return OLAP_SUCCESS;
}
OLAPStatus parse_conf_store_paths(const std::string& config_path,
std::vector<StorePath>* paths) {
try {
std::vector<std::string> item_vec;
boost::split(item_vec, config_path, boost::is_any_of(";"),
boost::token_compress_on);
for (auto& item : item_vec) {
StorePath path;
auto res = parse_root_path(item, &path);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "get config store path failed. path="
<< config_path;
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
paths->emplace_back(std::move(path));
}
} catch (...) {
LOG(WARNING) << "get config store path failed. path=" << config_path;
@ -67,5 +132,4 @@ OLAPStatus parse_conf_store_paths(
return OLAP_SUCCESS;
}
}

21
be/src/olap/options.h
View File

@ -26,19 +26,30 @@
namespace doris {
struct StorePath {
StorePath() : capacity_bytes(-1) { }
StorePath() : capacity_bytes(-1), storage_medium(TStorageMedium::HDD) {}
StorePath(const std::string& path_, int64_t capacity_bytes_)
: path(path_), capacity_bytes(capacity_bytes_) { }
: path(path_),
capacity_bytes(capacity_bytes_),
storage_medium(TStorageMedium::HDD) {}
StorePath(const std::string& path_, int64_t capacity_bytes_,
TStorageMedium::type storage_medium_)
: path(path_),
capacity_bytes(capacity_bytes_),
storage_medium(storage_medium_) {}
std::string path;
int64_t capacity_bytes;
TStorageMedium::type storage_medium;
};
OLAPStatus parse_conf_store_paths(const std::string& config_path, std::vector<StorePath>* path);
// parse a single root path of storage_root_path
OLAPStatus parse_root_path(const std::string& root_path, StorePath* path);
OLAPStatus parse_conf_store_paths(const std::string& config_path,
std::vector<StorePath>* path);
struct EngineOptions {
// list paths that tablet will be put into.
std::vector<StorePath> store_paths;
UniqueId backend_uid {0, 0};
UniqueId backend_uid{0, 0};
};
}

2
be/src/olap/storage_engine.cpp
View File

@ -146,7 +146,7 @@ void StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) {
OLAPStatus StorageEngine::open() {
// init store_map
for (auto& path : _options.store_paths) {
DataDir* store = new DataDir(path.path, path.capacity_bytes,
DataDir* store = new DataDir(path.path, path.capacity_bytes, path.storage_medium,
_tablet_manager.get(), _txn_manager.get());
auto st = store->init();
if (!st.ok()) {

63
be/src/tools/meta_tool.cpp
View File

@ -25,6 +25,7 @@
#include "common/status.h"
#include "util/file_utils.h"
#include "gen_cpp/olap_file.pb.h"
#include "olap/options.h"
#include "olap/data_dir.h"
#include "olap/tablet_meta_manager.h"
#include "olap/olap_define.h"
@ -45,8 +46,9 @@ using doris::FileUtils;
const std::string HEADER_PREFIX = "tabletmeta_";
DEFINE_string(root_path, "", "storage root path");
DEFINE_string(operation, "get_meta",
"valid operation: get_meta, flag, load_meta, delete_meta, show_meta");
DEFINE_string(
operation, "get_meta",
"valid operation: get_meta, flag, load_meta, delete_meta, show_meta");
DEFINE_int64(tablet_id, 0, "tablet_id for tablet meta");
DEFINE_int32(schema_hash, 0, "schema_hash for tablet meta");
DEFINE_string(json_meta_path, "", "absolute json meta file path");
@ -57,9 +59,13 @@ std::string get_usage(const std::string& progname) {
ss << progname << " is the Doris BE Meta tool.\n";
ss << "Stop BE first before use this tool.\n";
ss << "Usage:\n";
ss << "./meta_tool --operation=get_meta --root_path=/path/to/storage/path --tablet_id=tabletid --schema_hash=schemahash\n";
ss << "./meta_tool --operation=load_meta --root_path=/path/to/storage/path --json_meta_path=path\n";
ss << "./meta_tool --operation=delete_meta --root_path=/path/to/storage/path --tablet_id=tabletid --schema_hash=schemahash\n";
ss << "./meta_tool --operation=get_meta --root_path=/path/to/storage/path "
"--tablet_id=tabletid --schema_hash=schemahash\n";
ss << "./meta_tool --operation=load_meta --root_path=/path/to/storage/path "
"--json_meta_path=path\n";
ss << "./meta_tool --operation=delete_meta "
"--root_path=/path/to/storage/path --tablet_id=tabletid "
"--schema_hash=schemahash\n";
ss << "./meta_tool --operation=show_meta --pb_meta_path=path\n";
return ss.str();
}
@ -67,7 +73,7 @@ std::string get_usage(const std::string& progname) {
void show_meta() {
TabletMeta tablet_meta;
OLAPStatus s = tablet_meta.create_from_file(FLAGS_pb_meta_path);
if (s != OLAP_SUCCESS){
if (s != OLAP_SUCCESS) {
std::cout << "load pb meta file:" << FLAGS_pb_meta_path << " failed"
<< ", status:" << s << std::endl;
return;
@ -81,9 +87,10 @@ void show_meta() {
std::cout << json_meta << std::endl;
}
void get_meta(DataDir *data_dir) {
void get_meta(DataDir* data_dir) {
std::string value;
OLAPStatus s = TabletMetaManager::get_json_meta(data_dir, FLAGS_tablet_id, FLAGS_schema_hash, &value);
OLAPStatus s = TabletMetaManager::get_json_meta(data_dir, FLAGS_tablet_id,
FLAGS_schema_hash, &value);
if (s == doris::OLAP_ERR_META_KEY_NOT_FOUND) {
std::cout << "no tablet meta for tablet_id:" << FLAGS_tablet_id
<< ", schema_hash:" << FLAGS_schema_hash << std::endl;
@ -92,9 +99,10 @@ void get_meta(DataDir *data_dir) {
std::cout << value << std::endl;
}
void load_meta(DataDir *data_dir) {
void load_meta(DataDir* data_dir) {
// load json tablet meta into meta
OLAPStatus s = TabletMetaManager::load_json_meta(data_dir, FLAGS_json_meta_path);
OLAPStatus s =
TabletMetaManager::load_json_meta(data_dir, FLAGS_json_meta_path);
if (s != OLAP_SUCCESS) {
std::cout << "load meta failed, status:" << s << std::endl;
return;
@ -102,32 +110,29 @@ void load_meta(DataDir *data_dir) {
std::cout << "load meta successfully" << std::endl;
}
void delete_meta(DataDir *data_dir) {
OLAPStatus s = TabletMetaManager::remove(data_dir, FLAGS_tablet_id, FLAGS_schema_hash);
void delete_meta(DataDir* data_dir) {
OLAPStatus s =
TabletMetaManager::remove(data_dir, FLAGS_tablet_id, FLAGS_schema_hash);
if (s != OLAP_SUCCESS) {
std::cout << "delete tablet meta failed for tablet_id:" << FLAGS_tablet_id
<< ", schema_hash:" << FLAGS_schema_hash
std::cout << "delete tablet meta failed for tablet_id:"
<< FLAGS_tablet_id << ", schema_hash:" << FLAGS_schema_hash
<< ", status:" << s << std::endl;
return;
}
std::cout << "delete meta successfully" << std::endl;
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
std::string usage = get_usage(argv[0]);
gflags::SetUsageMessage(usage);
google::ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_operation == "show_meta") {
show_meta();
}
else {
} else {
// operations that need root path should be written here
std::set<std::string> valid_operations = {
"get_meta",
"load_meta",
"delete_meta"
};
std::set<std::string> valid_operations = {"get_meta", "load_meta",
"delete_meta"};
if (valid_operations.find(FLAGS_operation) == valid_operations.end()) {
std::cout << "invalid operation:" << FLAGS_operation << std::endl;
return -1;
@ -136,11 +141,19 @@ int main(int argc, char **argv) {
std::string root_path;
Status st = FileUtils::canonicalize(FLAGS_root_path, &root_path);
if (!st.ok()) {
std::cout << "invalid root path:" << FLAGS_root_path << ", error: " << st.to_string() << std::endl;
std::cout << "invalid root path:" << FLAGS_root_path
<< ", error: " << st.to_string() << std::endl;
return -1;
}
doris::StorePath path;
auto res = parse_root_path(root_path, &path);
if (res != OLAP_SUCCESS) {
std::cout << "parse root path failed:" << root_path << std::endl;
return -1;
}
std::unique_ptr<DataDir> data_dir(new (std::nothrow) DataDir(root_path));
std::unique_ptr<DataDir> data_dir(new (std::nothrow) DataDir(
path.path, path.capacity_bytes, path.storage_medium));
if (data_dir == nullptr) {
std::cout << "new data dir failed" << std::endl;
return -1;
@ -159,7 +172,7 @@ int main(int argc, char **argv) {
delete_meta(data_dir.get());
} else {
std::cout << "invalid operation:" << FLAGS_operation << "\n"
<< usage << std::endl;
<< usage << std::endl;
return -1;
}
}

1
be/test/olap/CMakeLists.txt
View File

@ -75,3 +75,4 @@ ADD_BE_TEST(page_cache_test)
ADD_BE_TEST(hll_test)
# ADD_BE_TEST(memtable_flush_executor_test)
ADD_BE_TEST(selection_vector_test)
ADD_BE_TEST(options_test)

120
be/test/olap/options_test.cpp Normal file
View File

@ -0,0 +1,120 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "olap/options.h"
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <string>
namespace doris {
void set_up() {
system("rm -rf ./test_run && mkdir -p ./test_run");
system("mkdir -p ./test_run/palo && mkdir -p ./test_run/palo.ssd");
}
void tear_down() { system("rm -rf ./test_run"); }
class OptionsTest : public testing::Test {
public:
OptionsTest() {}
virtual ~OptionsTest() {}
};
TEST_F(OptionsTest, parse_root_path) {
std::string path_prefix =
boost::filesystem::system_complete("./test_run").string();
std::string path1 = path_prefix + "/palo";
std::string path2 = path_prefix + "/palo.ssd";
std::string root_path;
StorePath path;
// /path<.extension>, <capacity>
{
root_path = path1;
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path1.c_str(), path.path.c_str());
ASSERT_EQ(-1, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::HDD, path.storage_medium);
}
{
root_path = path2;
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path2.c_str(), path.path.c_str());
ASSERT_EQ(-1, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::SSD, path.storage_medium);
}
{
root_path = path2 + ", 50";
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path2.c_str(), path.path.c_str());
ASSERT_EQ(50 * GB_EXCHANGE_BYTE, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::SSD, path.storage_medium);
}
// /path, <property>:<value>,...
{
root_path = path1 + ", capacity:50, medium: ssd";
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path1.c_str(), path.path.c_str());
ASSERT_EQ(50 * GB_EXCHANGE_BYTE, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::SSD, path.storage_medium);
}
{
root_path = path1 + ", medium: ssd, capacity:30";
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path1.c_str(), path.path.c_str());
ASSERT_EQ(30 * GB_EXCHANGE_BYTE, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::SSD, path.storage_medium);
}
{
root_path = path1 + " , medium: ssd, 60";
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path1.c_str(), path.path.c_str());
ASSERT_EQ(60 * GB_EXCHANGE_BYTE, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::SSD, path.storage_medium);
}
{
root_path = path1 + ", medium: ssd, 60, medium: hdd, capacity: 10";
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path1.c_str(), path.path.c_str());
ASSERT_EQ(10 * GB_EXCHANGE_BYTE, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::HDD, path.storage_medium);
}
{
root_path = path2 + ", medium: hdd, 60, capacity: 10";
ASSERT_EQ(OLAP_SUCCESS, parse_root_path(root_path, &path));
ASSERT_STREQ(path2.c_str(), path.path.c_str());
ASSERT_EQ(10 * GB_EXCHANGE_BYTE, path.capacity_bytes);
ASSERT_EQ(TStorageMedium::HDD, path.storage_medium);
}
}
} // namespace doris
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
int ret = doris::OLAP_SUCCESS;
doris::set_up();
ret = RUN_ALL_TESTS();
doris::tear_down();
return ret;
}

4
conf/be.conf
View File

@ -40,6 +40,10 @@ brpc_port = 8060
# /home/disk1/palo.HDD, capacity limit is 50GB, HDD;
# /home/disk2/palo.SSD, capacity limit is 1GB, SSD;
# /home/disk2/palo, capacity limit is disk capacity, HDD(default)
#
# you also can specify the properties by setting '<property>:<value>', seperate by ','
# property 'medium' has a higher priority than the extension of path
# storage_root_path = /home/disk1/palo,medium:ssd,capacity:50
storage_root_path = /home/disk1/palo;/home/disk2/palo
# Advanced configurations

1
run-ut.sh
View File

@ -249,6 +249,7 @@ ${DORIS_TEST_BINARY_DIR}/olap/row_cursor_test
${DORIS_TEST_BINARY_DIR}/olap/skiplist_test
${DORIS_TEST_BINARY_DIR}/olap/serialize_test
# ${DORIS_TEST_BINARY_DIR}/olap/memtable_flush_executor_test
${DORIS_TEST_BINARY_DIR}/olap/options_test
# Running routine load test
${DORIS_TEST_BINARY_DIR}/olap/tablet_meta_manager_test