[Compaction] Compaction show policy type and disk format (#4466)
Add more information in compaction show api 1、add cumulative policy type 2、format rowset total disk size
This commit is contained in:
ZhangYu0123
@ -114,6 +114,8 @@ public:
|
||||
int64_t current_cumulative_point,
|
||||
int64_t* cumulative_point) = 0;
|
||||
|
||||
/// Fetch cumulative policy name
|
||||
virtual std::string name() = 0;
|
||||
};
|
||||
|
||||
/// Num based cumulative compcation policy implemention. Num based policy which derives CumulativeCompactionPolicy is early
|
||||
@ -159,6 +161,7 @@ public:
|
||||
int64_t current_cumulative_point,
|
||||
uint32_t* score) override;
|
||||
|
||||
std::string name() { return CUMULATIVE_NUM_BASED_POLICY; }
|
||||
};
|
||||
|
||||
/// SizeBased cumulative compcation policy implemention. SizeBased policy which derives CumulativeCompactionPolicy is a optimized
|
||||
@ -213,6 +216,8 @@ public:
|
||||
int64_t current_cumulative_point,
|
||||
uint32_t* score) override;
|
||||
|
||||
std::string name() { return CUMULATIVE_SIZE_BASED_POLICY; }
|
||||
|
||||
private:
|
||||
/// calculate promotion size using current base rowset meta size and promition configs
|
||||
void _calc_promotion_size(RowsetMetaSharedPtr base_rowset_meta, int64_t* promotion_size);
|
||||
|
||||
@ -39,6 +39,7 @@
|
||||
#include "olap/tablet_meta_manager.h"
|
||||
#include "util/path_util.h"
|
||||
#include "util/time.h"
|
||||
#include "util/pretty_printer.h"
|
||||
|
||||
namespace doris {
|
||||
|
||||
@ -1036,7 +1037,10 @@ void Tablet::get_compaction_status(std::string* json_result) {
|
||||
// get snapshot version path json_doc
|
||||
_timestamped_version_tracker.get_stale_version_path_json_doc(path_arr);
|
||||
}
|
||||
|
||||
rapidjson::Value cumulative_policy_type;
|
||||
std::string policy_type_str = _cumulative_compaction_policy->name();
|
||||
cumulative_policy_type.SetString(policy_type_str.c_str(), policy_type_str.length(), root.GetAllocator());
|
||||
root.AddMember("cumulative policy type", cumulative_policy_type, root.GetAllocator());
|
||||
root.AddMember("cumulative point", _cumulative_point.load(), root.GetAllocator());
|
||||
rapidjson::Value cumu_value;
|
||||
std::string format_str = ToStringFromUnixMillis(_last_cumu_compaction_failure_millis.load());
|
||||
@ -1061,10 +1065,12 @@ void Tablet::get_compaction_status(std::string* json_result) {
|
||||
for (int i = 0; i < rowsets.size(); ++i) {
|
||||
const Version& ver = rowsets[i]->version();
|
||||
rapidjson::Value value;
|
||||
std::string version_str = strings::Substitute("[$0-$1] $2 $3 $4 $5",
|
||||
ver.first, ver.second, rowsets[i]->num_segments(), (delete_flags[i] ? "DELETE" : "DATA"),
|
||||
SegmentsOverlapPB_Name(rowsets[i]->rowset_meta()->segments_overlap()),
|
||||
rowsets[i]->rowset_meta()->total_disk_size());
|
||||
std::string disk_size =
|
||||
PrettyPrinter::print(rowsets[i]->rowset_meta()->total_disk_size(), TUnit::BYTES);
|
||||
std::string version_str = strings::Substitute(
|
||||
"[$0-$1] $2 $3 $4 $5", ver.first, ver.second, rowsets[i]->num_segments(),
|
||||
(delete_flags[i] ? "DELETE" : "DATA"),
|
||||
SegmentsOverlapPB_Name(rowsets[i]->rowset_meta()->segments_overlap()), disk_size);
|
||||
value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator());
|
||||
versions_arr.PushBack(value, versions_arr.GetAllocator());
|
||||
}
|
||||
|
||||
@ -53,26 +53,27 @@ If the tablet exists, the result is returned in JSON format:
|
||||
|
||||
```
|
||||
{
|
||||
"cumulative policy type": "NUM_BASED",
|
||||
"cumulative point": 50,
|
||||
"last cumulative failure time": "2019-12-16 18:13:43.224",
|
||||
"last base failure time": "2019-12-16 18:13:23.320",
|
||||
"last cumu success time": "2019-12-16 18:12:15.110",
|
||||
"last base success time": "2019-12-16 18:11:50.780",
|
||||
"rowsets": [
|
||||
"[0-48] 10 DATA OVERLAPPING",
|
||||
"[49-49] 2 DATA OVERLAPPING",
|
||||
"[50-50] 0 DELETE NONOVERLAPPING",
|
||||
"[51-51] 5 DATA OVERLAPPING"
|
||||
"[0-48] 10 DATA OVERLAPPING 574.00 MB",
|
||||
"[49-49] 2 DATA OVERLAPPING 574.00 B",
|
||||
"[50-50] 0 DELETE NONOVERLAPPING 574.00 B",
|
||||
"[51-51] 5 DATA OVERLAPPING 574.00 B"
|
||||
],
|
||||
"stale version path": [
|
||||
{
|
||||
"path id": "2",
|
||||
"last create time": "2019-12-16 18:11:15.110",
|
||||
"last create time": "2019-12-16 18:11:15.110 +0800",
|
||||
"path list": "2-> [0-24] -> [25-48]"
|
||||
},
|
||||
{
|
||||
"path id": "1",
|
||||
"last create time": "2019-12-16 18:13:15.110",
|
||||
"last create time": "2019-12-16 18:13:15.110 +0800",
|
||||
"path list": "1-> [25-40] -> [40-48]"
|
||||
}
|
||||
]
|
||||
@ -81,6 +82,7 @@ If the tablet exists, the result is returned in JSON format:
|
||||
|
||||
Explanation of results:
|
||||
|
||||
* cumulative policy type: The cumulative compaction policy type which is used by current tablet.
|
||||
* cumulative point: The version boundary between base and cumulative compaction. Versions before (excluding) points are handled by base compaction. Versions after (inclusive) are handled by cumulative compaction.
|
||||
* last cumulative failure time: The time when the last cumulative compaction failed. After 10 minutes by default, cumulative compaction is attempted on the this tablet again.
|
||||
* last base failure time: The time when the last base compaction failed. After 10 minutes by default, base compaction is attempted on the this tablet again.
|
||||
|
||||
@ -53,26 +53,27 @@ curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&sc
|
||||
|
||||
```
|
||||
{
|
||||
"cumulative policy type": "NUM_BASED",
|
||||
"cumulative point": 50,
|
||||
"last cumulative failure time": "2019-12-16 18:13:43.224",
|
||||
"last base failure time": "2019-12-16 18:13:23.320",
|
||||
"last cumu success time": ,
|
||||
"last base success time": "2019-12-16 18:11:50.780",
|
||||
"rowsets": [
|
||||
"[0-48] 10 DATA OVERLAPPING",
|
||||
"[49-49] 2 DATA OVERLAPPING",
|
||||
"[50-50] 0 DELETE NONOVERLAPPING",
|
||||
"[51-51] 5 DATA OVERLAPPING"
|
||||
"[0-48] 10 DATA OVERLAPPING 574.00 MB",
|
||||
"[49-49] 2 DATA OVERLAPPING 574.00 B",
|
||||
"[50-50] 0 DELETE NONOVERLAPPING 574.00 B",
|
||||
"[51-51] 5 DATA OVERLAPPING 574.00 B"
|
||||
],
|
||||
"stale version path": [
|
||||
{
|
||||
"path id": "2",
|
||||
"last create time": "2019-12-16 18:11:15.110",
|
||||
"last create time": "2019-12-16 18:11:15.110 +0800",
|
||||
"path list": "2-> [0-24] -> [25-48]"
|
||||
},
|
||||
{
|
||||
"path id": "1",
|
||||
"last create time": "2019-12-16 18:13:15.110",
|
||||
"last create time": "2019-12-16 18:13:15.110 +0800",
|
||||
"path list": "1-> [25-40] -> [40-48]"
|
||||
}
|
||||
]
|
||||
@ -81,6 +82,7 @@ curl -X GET http://be_host:webserver_port/api/compaction/show?tablet_id=xxxx\&sc
|
||||
|
||||
结果说明：
|
||||
|
||||
* cumulative policy type：当前tablet所使用的 cumulative compaction 策略。
|
||||
* cumulative point：base 和 cumulative compaction 的版本分界线。在 point（不含）之前的版本由 base compaction 处理。point（含）之后的版本由 cumulative compaction 处理。
|
||||
* last cumulative failure time：上一次尝试 cumulative compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 cumulative compaction。
|
||||
* last base failure time：上一次尝试 base compaction 失败的时间。默认 10min 后才会再次尝试对该 tablet 做 base compaction。
|
||||
|
||||
Reference in New Issue
Block a user