Optimize compaction strategy of tablet on BE (#2473)
The current compaction selection strategy and cumulative point update logic will cause the cumulative compaction to not work, and all compaction tasks will be completed only by the base compaction. This can cause a large number of data versions to pile up. In the current cumulative point update logic, when a cumulative cannot select enough number of rowsets, it will directly increase the cumulative point. Therefore, when the data version generates the same speed as the cumulative compaction polling, it will cause the cumulative point to continuously increase without triggering the cumulative compaction. The new strategy mainly modifies the update logic of cumulative point to ensure that the above problems do not occur. At the same time, the new strategy also takes into account the problem that compaction cannot be performed if cumulative points stagnate for a long time. Cumulative points will be forced to increase through threshold settings to ensure that compaction has a chance to execute. Also add a new HTTP API to view the compaction status of specified tablet. See `compaction-action.md` for details.
This commit is contained in:
Mingyu Chen
@ -238,7 +238,7 @@ namespace config {
|
||||
CONF_Int32(base_compaction_num_threads, "1");
|
||||
CONF_Int32(base_compaction_num_threads_per_disk, "1");
|
||||
CONF_Double(base_cumulative_delta_ratio, "0.3");
|
||||
CONF_Int64(base_compaction_interval_seconds_since_last_operation, "604800");
|
||||
CONF_Int64(base_compaction_interval_seconds_since_last_operation, "86400");
|
||||
CONF_Int32(base_compaction_write_mbytes_per_sec, "5");
|
||||
|
||||
// cumulative compaction policy: max delta file's size unit:B
|
||||
|
||||
@ -45,6 +45,7 @@ add_library(Webserver STATIC
|
||||
action/metrics_action.cpp
|
||||
action/stream_load.cpp
|
||||
action/meta_action.cpp
|
||||
action/compaction_action.cpp
|
||||
# action/multi_start.cpp
|
||||
# action/multi_show.cpp
|
||||
# action/multi_commit.cpp
|
||||
|
||||
82
be/src/http/action/compaction_action.cpp
Normal file
82
be/src/http/action/compaction_action.cpp
Normal file
@ -0,0 +1,82 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "http/action/compaction_action.h"
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
#include "http/http_channel.h"
|
||||
#include "http/http_request.h"
|
||||
#include "http/http_response.h"
|
||||
#include "http/http_headers.h"
|
||||
#include "http/http_status.h"
|
||||
|
||||
#include "olap/storage_engine.h"
|
||||
#include "olap/olap_define.h"
|
||||
#include "olap/tablet.h"
|
||||
#include "common/logging.h"
|
||||
#include "gutil/strings/substitute.h"
|
||||
#include "util/json_util.h"
|
||||
|
||||
namespace doris {
|
||||
|
||||
const static std::string HEADER_JSON = "application/json";
|
||||
|
||||
// for viewing the compaction status
|
||||
Status CompactionAction::_handle_show_compaction(HttpRequest *req, std::string* json_result) {
|
||||
std::string req_tablet_id = req->param(TABLET_ID_KEY);
|
||||
std::string req_schema_hash = req->param(TABLET_SCHEMA_HASH_KEY);
|
||||
if (req_tablet_id == "" && req_schema_hash == "") {
|
||||
// TODO(cmy): View the overall compaction status
|
||||
return Status::NotSupported("The overall compaction status is not supported yet");
|
||||
} else if (req_tablet_id == "" || req_schema_hash == "") {
|
||||
return Status::InvalidArgument("Missing tablet id or schema hash");
|
||||
}
|
||||
|
||||
uint64_t tablet_id = std::stoull(req_tablet_id);
|
||||
uint32_t schema_hash = std::stoul(req_schema_hash);
|
||||
|
||||
TabletSharedPtr tablet = StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
|
||||
if (tablet == nullptr) {
|
||||
return Status::NotFound("Tablet not found");
|
||||
}
|
||||
|
||||
OLAPStatus s = tablet->get_compaction_status(json_result);
|
||||
if (s != OLAP_SUCCESS) {
|
||||
return Status::InternalError(strings::Substitute("failed to get tablet compaction status. res $0", s));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void CompactionAction::handle(HttpRequest *req) {
|
||||
req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
|
||||
|
||||
if (_type == CompactionActionType::SHOW_INFO) {
|
||||
std::string json_result;
|
||||
Status st = _handle_show_compaction(req, &json_result);
|
||||
if (!st.ok()) {
|
||||
HttpChannel::send_reply(req, HttpStatus::OK, to_json(st));
|
||||
} else {
|
||||
HttpChannel::send_reply(req, HttpStatus::OK, json_result);
|
||||
}
|
||||
} else {
|
||||
HttpChannel::send_reply(req, HttpStatus::OK, to_json(Status::NotSupported("Action not supported")));
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace doris
|
||||
49
be/src/http/action/compaction_action.h
Normal file
49
be/src/http/action/compaction_action.h
Normal file
@ -0,0 +1,49 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "http/http_handler.h"
|
||||
#include "common/status.h"
|
||||
|
||||
namespace doris {
|
||||
|
||||
|
||||
enum CompactionActionType {
|
||||
SHOW_INFO = 1,
|
||||
RUN_COMPACTION = 2
|
||||
};
|
||||
|
||||
// This action is used for viewing the compaction status.
|
||||
// See compaction-action.md for details.
|
||||
class CompactionAction : public HttpHandler {
|
||||
public:
|
||||
CompactionAction(CompactionActionType type) : _type(type) {}
|
||||
|
||||
virtual ~CompactionAction() {}
|
||||
|
||||
void handle(HttpRequest *req) override;
|
||||
|
||||
private:
|
||||
Status _handle_show_compaction(HttpRequest *req, std::string* json_result);
|
||||
|
||||
private:
|
||||
CompactionActionType _type;
|
||||
};
|
||||
|
||||
} // end namespace doris
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#include "gutil/strings/substitute.h"
|
||||
#include "olap/compaction.h"
|
||||
#include "olap/rowset/rowset_factory.h"
|
||||
#include "util/time.h"
|
||||
|
||||
using std::vector;
|
||||
|
||||
@ -76,6 +77,14 @@ OLAPStatus Compaction::do_compaction() {
|
||||
// 4. modify rowsets in memory
|
||||
RETURN_NOT_OK(modify_rowsets());
|
||||
|
||||
// 5. update last success compaction time
|
||||
int64_t now = UnixMillis();
|
||||
if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION) {
|
||||
_tablet->set_last_cumu_compaction_success_time(now);
|
||||
} else {
|
||||
_tablet->set_last_base_compaction_success_time(now);
|
||||
}
|
||||
|
||||
LOG(INFO) << "succeed to do " << compaction_name()
|
||||
<< ". tablet=" << _tablet->full_name()
|
||||
<< ", output_version=" << _output_version.first
|
||||
|
||||
@ -76,13 +76,18 @@ OLAPStatus CumulativeCompaction::pick_rowsets_to_compact() {
|
||||
|
||||
std::vector<RowsetSharedPtr> transient_rowsets;
|
||||
size_t num_overlapping_segments = 0;
|
||||
// the last delete version we meet when traversing candidate_rowsets
|
||||
Version last_delete_version { -1, -1 };
|
||||
|
||||
// traverse rowsets from begin to penultimate rowset.
|
||||
// Because VersionHash will calculated from chosen rowsets.
|
||||
// If ultimate singleton rowset is chosen, VersionHash
|
||||
// will be different from the value recorded in FE.
|
||||
// So the ultimate singleton rowset is revserved.
|
||||
for (size_t i = 0; i < candidate_rowsets.size() - 1; ++i) {
|
||||
// VersionHash will calculated from chosen rowsets.
|
||||
// If ultimate singleton rowset is chosen, VersionHash
|
||||
// will be different from the value recorded in FE.
|
||||
// So the ultimate singleton rowset is revserved.
|
||||
RowsetSharedPtr rowset = candidate_rowsets[i];
|
||||
if (_tablet->version_for_delete_predicate(rowset->version())) {
|
||||
last_delete_version = rowset->version();
|
||||
if (num_overlapping_segments >= config::min_cumulative_compaction_num_singleton_deltas) {
|
||||
_input_rowsets = transient_rowsets;
|
||||
break;
|
||||
@ -108,12 +113,35 @@ OLAPStatus CumulativeCompaction::pick_rowsets_to_compact() {
|
||||
if (num_overlapping_segments >= config::min_cumulative_compaction_num_singleton_deltas) {
|
||||
_input_rowsets = transient_rowsets;
|
||||
}
|
||||
|
||||
|
||||
// Cumulative compaction will process with at least 2 rowsets.
|
||||
// So when there is no rowset or only 1 rowset being chosen, we should return OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSIONS:
|
||||
if (_input_rowsets.size() <= 1) {
|
||||
// There are no suitable rowsets choosed to do cumulative compaction.
|
||||
// Under this circumstance, cumulative_point should be set.
|
||||
// Otherwise, the next round will not choose rowsets.
|
||||
_tablet->set_cumulative_layer_point(candidate_rowsets.back()->start_version());
|
||||
if (last_delete_version.first != -1) {
|
||||
// we meet a delete version, should increase the cumulative point to let base compaction handle the delete version.
|
||||
// plus 1 to skip the delete version.
|
||||
// NOTICE: after that, the cumulative point may be larger than max version of this tablet, but it doen't matter.
|
||||
_tablet->set_cumulative_layer_point(last_delete_version.first + 1);
|
||||
return OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSIONS;
|
||||
}
|
||||
|
||||
// we did not meet any delete version. which means num_overlapping_segments is not enough to do cumulative compaction.
|
||||
// We should wait until there are more rowsets to come, and keep the cumulative point unchanged.
|
||||
// But in order to avoid the stall of compaction because no new rowset arrives later, we should increase
|
||||
// the cumulative point after waiting for a long time, to ensure that the base compaction can continue.
|
||||
|
||||
// check both last success time of base and cumulative compaction
|
||||
int64_t last_cumu_compaction_success_time = _tablet->last_cumu_compaction_success_time();
|
||||
int64_t last_base_compaction_success_time = _tablet->last_base_compaction_success_time();
|
||||
|
||||
int64_t interval_threshold = config::base_compaction_interval_seconds_since_last_operation;
|
||||
int64_t now = time(NULL);
|
||||
int64_t cumu_interval = now - last_cumu_compaction_success_time;
|
||||
int64_t base_interval = now - last_base_compaction_success_time;
|
||||
if (cumu_interval > interval_threshold && base_interval > interval_threshold) {
|
||||
_tablet->set_cumulative_layer_point(candidate_rowsets.back()->start_version());
|
||||
}
|
||||
|
||||
return OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSIONS;
|
||||
}
|
||||
|
||||
|
||||
@ -42,7 +42,6 @@ protected:
|
||||
return ReaderType::READER_CUMULATIVE_COMPACTION;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
int64_t _cumulative_rowset_size_threshold;
|
||||
|
||||
|
||||
@ -203,7 +203,6 @@ struct Version {
|
||||
|
||||
typedef std::vector<Version> Versions;
|
||||
|
||||
|
||||
// used for hash-struct of hash_map<Version, Rowset*>.
|
||||
struct HashOfVersion {
|
||||
size_t operator()(const Version& version) const {
|
||||
|
||||
@ -544,7 +544,7 @@ void StorageEngine::perform_cumulative_compaction(DataDir* data_dir) {
|
||||
|
||||
OLAPStatus res = cumulative_compaction.compact();
|
||||
if (res != OLAP_SUCCESS) {
|
||||
best_tablet->set_last_compaction_failure_time(UnixMillis());
|
||||
best_tablet->set_last_cumu_compaction_failure_time(UnixMillis());
|
||||
if (res != OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSIONS) {
|
||||
DorisMetrics::cumulative_compaction_request_failed.increment(1);
|
||||
LOG(WARNING) << "failed to do cumulative compaction. res=" << res
|
||||
@ -553,7 +553,7 @@ void StorageEngine::perform_cumulative_compaction(DataDir* data_dir) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
best_tablet->set_last_compaction_failure_time(0);
|
||||
best_tablet->set_last_cumu_compaction_failure_time(0);
|
||||
}
|
||||
|
||||
void StorageEngine::perform_base_compaction(DataDir* data_dir) {
|
||||
@ -566,7 +566,7 @@ void StorageEngine::perform_base_compaction(DataDir* data_dir) {
|
||||
BaseCompaction base_compaction(best_tablet);
|
||||
OLAPStatus res = base_compaction.compact();
|
||||
if (res != OLAP_SUCCESS) {
|
||||
best_tablet->set_last_compaction_failure_time(UnixMillis());
|
||||
best_tablet->set_last_base_compaction_failure_time(UnixMillis());
|
||||
if (res != OLAP_ERR_BE_NO_SUITABLE_VERSION) {
|
||||
DorisMetrics::base_compaction_request_failed.increment(1);
|
||||
LOG(WARNING) << "failed to init base compaction. res=" << res
|
||||
@ -574,7 +574,7 @@ void StorageEngine::perform_base_compaction(DataDir* data_dir) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
best_tablet->set_last_compaction_failure_time(0);
|
||||
best_tablet->set_last_base_compaction_failure_time(0);
|
||||
}
|
||||
|
||||
void StorageEngine::get_cache_status(rapidjson::Document* document) const {
|
||||
|
||||
@ -26,6 +26,8 @@
|
||||
#include <set>
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <rapidjson/prettywriter.h>
|
||||
#include <rapidjson/stringbuffer.h>
|
||||
|
||||
#include "olap/data_dir.h"
|
||||
#include "olap/olap_common.h"
|
||||
@ -65,7 +67,11 @@ Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir)
|
||||
_schema(tablet_meta->tablet_schema()),
|
||||
_data_dir(data_dir),
|
||||
_is_bad(false),
|
||||
_last_compaction_failure_time(0) {
|
||||
_last_cumu_compaction_failure_time(0),
|
||||
_last_base_compaction_failure_time(0),
|
||||
_last_cumu_compaction_success_time(0),
|
||||
_last_base_compaction_success_time(0) {
|
||||
|
||||
_tablet_path.append(_data_dir->path());
|
||||
_tablet_path.append(DATA_PREFIX);
|
||||
_tablet_path.append("/");
|
||||
@ -699,7 +705,6 @@ const uint32_t Tablet::calc_base_compaction_score() const {
|
||||
base_rowset_exist = true;
|
||||
}
|
||||
}
|
||||
score = score < config::base_compaction_num_cumulative_deltas ? 0 : score;
|
||||
|
||||
// base不存在可能是tablet正在做alter table，先不选它，设score=0
|
||||
return base_rowset_exist ? score : 0;
|
||||
@ -1035,6 +1040,64 @@ void Tablet::pick_candicate_rowsets_to_base_compaction(std::vector<RowsetSharedP
|
||||
}
|
||||
}
|
||||
|
||||
OLAPStatus Tablet::get_compaction_status(std::string* json_result) {
|
||||
rapidjson::Document root;
|
||||
root.SetObject();
|
||||
|
||||
auto rowset_version_map = std::map<Version, bool, std::function<bool(const Version&, const Version&)>>{
|
||||
[](const Version& lhs, const Version& rhs)
|
||||
{
|
||||
if (lhs.first < rhs.first) return true;
|
||||
if (lhs.first == rhs.first) return lhs.second < rhs.second;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
{
|
||||
ReadLock rdlock(&_meta_lock);
|
||||
for (auto& it : _rs_version_map) {
|
||||
rowset_version_map[it.first] = version_for_delete_predicate(it.first);
|
||||
}
|
||||
root.AddMember("cumulative point", _cumulative_point.load(), root.GetAllocator());
|
||||
rapidjson::Value cumu_value;
|
||||
std::string format_str = ToStringFromUnixMillis(_last_cumu_compaction_failure_time.load());
|
||||
cumu_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
|
||||
root.AddMember("last cumulative failure time", cumu_value, root.GetAllocator());
|
||||
rapidjson::Value base_value;
|
||||
format_str = ToStringFromUnixMillis(_last_base_compaction_failure_time.load());
|
||||
base_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
|
||||
root.AddMember("last base failure time", base_value, root.GetAllocator());
|
||||
rapidjson::Value cumu_success_value;
|
||||
format_str = ToStringFromUnixMillis(_last_cumu_compaction_success_time.load());
|
||||
cumu_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
|
||||
root.AddMember("last cumulative success time", cumu_success_value, root.GetAllocator());
|
||||
rapidjson::Value base_success_value;
|
||||
format_str = ToStringFromUnixMillis(_last_base_compaction_success_time.load());
|
||||
base_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
|
||||
root.AddMember("last base success time", base_success_value, root.GetAllocator());
|
||||
}
|
||||
// std::sort(all_rowsets.begin(), all_rowsets.end(), Rowset::comparator);
|
||||
|
||||
// print all rowsets' version as an array
|
||||
rapidjson::Document versions_arr;
|
||||
versions_arr.SetArray();
|
||||
for (auto& it : rowset_version_map) {
|
||||
rapidjson::Value value;
|
||||
std::string version_str = strings::Substitute("[$0-$1] $2",
|
||||
it.first.first, it.first.second, (it.second ? "DELETE" : ""));
|
||||
value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator());
|
||||
versions_arr.PushBack(value, versions_arr.GetAllocator());
|
||||
}
|
||||
root.AddMember("versions", versions_arr, root.GetAllocator());
|
||||
|
||||
// to json string
|
||||
rapidjson::StringBuffer strbuf;
|
||||
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
|
||||
root.Accept(writer);
|
||||
*json_result = std::string(strbuf.GetString());
|
||||
|
||||
return OLAP_SUCCESS;
|
||||
}
|
||||
|
||||
OLAPStatus Tablet::do_tablet_meta_checkpoint() {
|
||||
WriteLock store_lock(&_meta_store_lock);
|
||||
if (_newly_created_rowset_num == 0) {
|
||||
|
||||
@ -212,11 +212,17 @@ public:
|
||||
void set_io_error();
|
||||
void set_bad(bool is_bad) { _is_bad = is_bad; }
|
||||
|
||||
int64_t last_compaction_failure_time() { return _last_compaction_failure_time; }
|
||||
int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_time; }
|
||||
void set_last_cumu_compaction_failure_time(int64_t time) { _last_cumu_compaction_failure_time = time; }
|
||||
|
||||
void set_last_compaction_failure_time(int64_t time) {
|
||||
_last_compaction_failure_time = time;
|
||||
}
|
||||
int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_time; }
|
||||
void set_last_base_compaction_failure_time(int64_t time) { _last_base_compaction_failure_time = time; }
|
||||
|
||||
int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_time; }
|
||||
void set_last_cumu_compaction_success_time(int64_t time) { _last_cumu_compaction_success_time = time; }
|
||||
|
||||
int64_t last_base_compaction_success_time() { return _last_base_compaction_success_time; }
|
||||
void set_last_base_compaction_success_time(int64_t time) { _last_base_compaction_success_time = time; }
|
||||
|
||||
void delete_all_files();
|
||||
|
||||
@ -249,6 +255,9 @@ public:
|
||||
|
||||
OLAPStatus generate_tablet_meta_copy(TabletMetaSharedPtr new_tablet_meta);
|
||||
|
||||
// return a json string to show the compaction status of this tablet
|
||||
OLAPStatus get_compaction_status(std::string* json_result);
|
||||
|
||||
private:
|
||||
OLAPStatus _init_once_action();
|
||||
void _print_missed_versions(const std::vector<Version>& missed_versions) const;
|
||||
@ -277,7 +286,10 @@ private:
|
||||
std::unordered_map<Version, RowsetSharedPtr, HashOfVersion> _inc_rs_version_map;
|
||||
|
||||
std::atomic<bool> _is_bad; // if this tablet is broken, set to true. default is false
|
||||
std::atomic<int64_t> _last_compaction_failure_time; // timestamp of last compaction failure
|
||||
std::atomic<int64_t> _last_cumu_compaction_failure_time; // timestamp of last cumulative compaction failure
|
||||
std::atomic<int64_t> _last_base_compaction_failure_time; // timestamp of last base compaction failure
|
||||
std::atomic<int64_t> _last_cumu_compaction_success_time; // timestamp of last cumu compaction success
|
||||
std::atomic<int64_t> _last_base_compaction_success_time; // timestamp of last base compaction success
|
||||
|
||||
std::atomic<int64_t> _cumulative_point;
|
||||
std::atomic<int32_t> _newly_created_rowset_num;
|
||||
|
||||
@ -723,7 +723,13 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (now - table_ptr->last_compaction_failure_time() <= config::min_compaction_failure_interval_sec * 1000) {
|
||||
int64_t last_failure_time = table_ptr->last_cumu_compaction_failure_time();
|
||||
if (compaction_type == CompactionType::BASE_COMPACTION) {
|
||||
last_failure_time = table_ptr->last_base_compaction_failure_time();
|
||||
}
|
||||
if (now - last_failure_time <= config::min_compaction_failure_interval_sec * 1000) {
|
||||
VLOG(1) << "last " << (compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative")
|
||||
<< " compaction failure time is: " << last_failure_time << ", tablet: " << table_ptr->tablet_id();
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#include "service/http_service.h"
|
||||
|
||||
#include "http/action/checksum_action.h"
|
||||
#include "http/action/compaction_action.h"
|
||||
#include "http/action/health_action.h"
|
||||
#include "http/action/meta_action.h"
|
||||
#include "http/action/metrics_action.h"
|
||||
@ -112,6 +113,12 @@ Status HttpService::start() {
|
||||
_ev_http_server->register_handler(HttpMethod::GET, "/api/snapshot", snapshot_action);
|
||||
#endif
|
||||
|
||||
// 2 compaction actions
|
||||
CompactionAction* show_compaction_action = new CompactionAction(CompactionActionType::SHOW_INFO);
|
||||
_ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/show", show_compaction_action);
|
||||
CompactionAction* run_compaction_action = new CompactionAction(CompactionActionType::RUN_COMPACTION);
|
||||
_ev_http_server->register_handler(HttpMethod::POST, "/api/compaction/run", run_compaction_action);
|
||||
|
||||
RETURN_IF_ERROR(_ev_http_server->start());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
@ -0,0 +1,79 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Compaction Action
|
||||
|
||||
该 API 用于查看某个 BE 节点总体的 compaction 状态，或者指定 tablet 的 compaction 状态。也可以用于手动触发 Compaction。
|
||||
|
||||
## 查看 Compaction 状态
|
||||
|
||||
### 节点整体 compaction 状态
|
||||
|
||||
(TODO)
|
||||
|
||||
### 指定 tablet 的 compaction 状态
|
||||
|
||||
```
|
||||
curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&schema_hash=yyyy
|
||||
```
|
||||
|
||||
若 tablet 不存在，返回 JSON 格式的错误：
|
||||
|
||||
```
|
||||
{
|
||||
"status": "Fail",
|
||||
"msg": "Tablet not found"
|
||||
}
|
||||
```
|
||||
|
||||
若 tablet 存在，则返回 JSON 格式的结果:
|
||||
|
||||
```
|
||||
{
|
||||
"cumulative point": 50,
|
||||
"last cumulative failure time": "2019-12-16 18:13:43.224",
|
||||
"last base failure time": "2019-12-16 18:13:23.320",
|
||||
"last cumu success time": "2019-12-16 18:12:15.110",
|
||||
"last base success time": "2019-12-16 18:11:50.780",
|
||||
"versions": [
|
||||
"[0-48] ",
|
||||
"[49-49] ",
|
||||
"[50-50] DELETE",
|
||||
"[51-51] "
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
结果说明：
|
||||
|
||||
* cumulative point：base 和 cumulative compaction 的版本分界线。在 point（不含）之前的版本由 base compaction 处理。point（含）之后的版本由 cumulative compaction 处理。
|
||||
* last cumulative failure time：上一次尝试 cumulative compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 cumulative compaction。
|
||||
* last base failure time：上一次尝试 base compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 base compaction。
|
||||
* versions：该 tablet 当前的数据版本集合。其中后 `DELETE` 后缀的表示 delete 版本。
|
||||
|
||||
### 示例
|
||||
|
||||
```
|
||||
curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schema_hash=1294206575
|
||||
```
|
||||
|
||||
## 手动触发 Compaction
|
||||
|
||||
(TODO)
|
||||
|
||||
@ -0,0 +1,78 @@
|
||||
<!-
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
->
|
||||
|
||||
# Compaction Action
|
||||
|
||||
This API is used to view the overall compaction status of a BE node or the compaction status of a specified tablet. It can also be used to manually trigger Compaction.
|
||||
|
||||
## View Compaction status
|
||||
|
||||
### The overall compaction status of the node
|
||||
|
||||
(TODO)
|
||||
|
||||
### Specify the compaction status of the tablet
|
||||
|
||||
```
|
||||
curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&schema_hash=yyyy
|
||||
```
|
||||
|
||||
If the tablet does not exist, an error in JSON format is returned:
|
||||
|
||||
```
|
||||
{
|
||||
"status": "Fail",
|
||||
"msg": "Tablet not found"
|
||||
}
|
||||
```
|
||||
|
||||
If the tablet exists, the result is returned in JSON format:
|
||||
|
||||
```
|
||||
{
|
||||
"cumulative point": 50,
|
||||
"last cumulative failure time": "2019-12-16 18:13:43.224",
|
||||
"last base failure time": "2019-12-16 18:13:23.320",
|
||||
"last cumu success time": "2019-12-16 18:12:15.110",
|
||||
"last base success time": "2019-12-16 18:11:50.780",
|
||||
"versions": [
|
||||
"[0-48] ",
|
||||
"[49-49] ",
|
||||
"[50-50] DELETE",
|
||||
"[51-51] "
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Explanation of results:
|
||||
|
||||
* cumulative point: The version boundary between base and cumulative compaction. Versions before (excluding) points are handled by base compaction. Versions after (inclusive) are handled by cumulative compaction.
|
||||
* last cumulative failure time: The time when the last cumulative compaction failed. After 10 minutes by default, cumulative compaction is attempted on the this tablet again.
|
||||
* last base failure time: The time when the last base compaction failed. After 10 minutes by default, base compaction is attempted on the this tablet again.
|
||||
* versions: The current data version collection of this tablet. The `DELETE` suffix indicates the delete version.
|
||||
|
||||
### Examples
|
||||
|
||||
```
|
||||
curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schema_hash=1294206575
|
||||
```
|
||||
|
||||
## Manually trigger Compaction
|
||||
|
||||
(TODO)
|
||||
Reference in New Issue
Block a user