hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] optimizer statistics gather enhance

Browse Source 
Co-authored-by: Larry955 <1412857955@qq.com>
Co-authored-by: wangt1xiuyi <13547954130@163.com>
This commit is contained in:
obdev
2023-04-28 13:11:58 +00:00
committed by ob-robot
parent 35c1be5aa1
commit 642f1c7d84
130 changed files with 8572 additions and 1563 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
src/share/stat/ob_opt_stat_sql_service.cpp
View File

@ -32,7 +32,6 @@
#include "observer/ob_server_struct.h"
#include "share/stat/ob_opt_column_stat.h"
#include "share/stat/ob_opt_table_stat.h"
#include "share/stat/ob_column_stat.h"
#include "lib/charset/ob_charset.h"
#include "share/stat/ob_opt_stat_monitor_manager.h"
@ -219,6 +218,25 @@
"user_stats = VALUES(user_stats);"
// TODO DAISI, add a sys_func to merge NDV by llc.
#define INSERT_TASK_OPT_STAT_GATHER_SQL "INSERT INTO %s(tenant_id," \
"task_id," \
"type," \
"ret_code," \
"failed_count,"\
"table_count," \
"start_time," \
"end_time) VALUES (%s);"
#define INSERT_TABLE_OPT_STAT_GATHER_SQL "INSERT INTO %s(tenant_id," \
"task_id," \
"table_id," \
"ret_code," \
"start_time," \
"end_time," \
"memory_used," \
"stat_refresh_failed_list," \
"properties) VALUES (%s);"
#define DEFINE_SQL_CLIENT_RETRY_WEAK_FOR_STAT(sql_client, table_name) \
const int64_t snapshot_timestamp = OB_INVALID_TIMESTAMP; \
@ -887,7 +905,7 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id,
int64_t llc_hex_size = 0;
if (OB_FAIL(get_valid_obj_str(stat.get_min_value(), min_meta, allocator, min_str, print_params)) ||
OB_FAIL(get_valid_obj_str(stat.get_max_value(), max_meta, allocator, max_str, print_params))) {
LOG_WARN("failed to get valid obj str", K(ret));
LOG_WARN("failed to get valid obj str", K(stat.get_min_value()), K(stat.get_max_value()));
} else if (OB_FAIL(get_obj_binary_hex_str(stat.get_min_value(), allocator, b_min_str)) ||
OB_FAIL(get_obj_binary_hex_str(stat.get_max_value(), allocator, b_max_str))) {
LOG_WARN("failed to convert obj to str", K(ret));
@ -1141,11 +1159,11 @@ int ObOptStatSqlService::get_obj_str(const ObObj &obj,
ObObjPrintParams copy_print_params = print_params;
copy_print_params.cs_type_ = obj.get_collation_type();
if (OB_FAIL(obj.print_varchar_literal(buf, buf_len, pos, copy_print_params))) {
LOG_WARN("failed to print sql literal", K(ret));
LOG_WARN("failed to print sql literal", K(obj));
} else { /*do nothing*/ }
} else if (obj.is_valid_type()) {
if (OB_FAIL(obj.print_sql_literal(buf, buf_len, pos, print_params))) {
LOG_WARN("failed to print_sql_literal", K(ret));
LOG_WARN("failed to print_sql_literal", K(obj));
} else { /*do nothing*/ }
} else {
ret = OB_ERR_UNEXPECTED;
@ -1400,7 +1418,7 @@ int ObOptStatSqlService::fill_column_stat(ObIAllocator &allocator,
common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length());
// decompress llc bitmap;
char *decomp_buf = NULL ;
int64_t decomp_size = ObColumnStat::NUM_LLC_BUCKET;
int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET;
const int64_t bitmap_size = hex_str.length() / 2;
if (OB_FAIL(get_decompressed_llc_bitmap(allocator, bitmap_buf,
bitmap_size, decomp_buf, decomp_size))) {
@ -1576,10 +1594,10 @@ int ObOptStatSqlService::get_decompressed_llc_bitmap(ObIAllocator &allocator,
int64_t &bitmap_size)
{
int ret = OB_SUCCESS;
const int64_t max_bitmap_size = ObColumnStat::NUM_LLC_BUCKET; // max size of uncompressed buffer.
const int64_t max_bitmap_size = ObOptColumnStat::NUM_LLC_BUCKET; // max size of uncompressed buffer.
ObCompressor* compressor = NULL;
if (comp_size >= ObColumnStat::NUM_LLC_BUCKET) {
if (comp_size >= ObOptColumnStat::NUM_LLC_BUCKET) {
// not compressed bitmap, use directly;
bitmap_buf = const_cast<char*>(comp_buf);
bitmap_size = comp_size;
@ -2131,6 +2149,121 @@ int ObOptStatSqlService::gen_tablet_list_str(const ObIArray<ObTabletID> &all_tab
return ret;
}
int ObOptStatSqlService::update_opt_stat_task_stat(const ObOptStatTaskInfo &task_info)
{
int ret = OB_SUCCESS;
ObSqlString raw_sql;
ObSqlString value_str;
int64_t affected_rows = 0;
const uint64_t tenant_id = gen_meta_tenant_id(task_info.tenant_id_);
if (OB_FAIL(get_gather_stat_task_value(task_info, value_str))) {
LOG_WARN("failed to get gather stat values list", K(ret));
} else if (OB_FAIL(raw_sql.append_fmt(INSERT_TASK_OPT_STAT_GATHER_SQL,
share::OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TNAME,
value_str.ptr()))) {
LOG_WARN("failed to append fmt", K(ret), K(raw_sql));
} else {
ObMySQLTransaction trans;
LOG_TRACE("sql string of update opt stat task stat", K(raw_sql));
if (OB_FAIL(trans.start(mysql_proxy_, tenant_id))) {
LOG_WARN("fail to start transaction", K(ret), K(tenant_id));
} else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) {
LOG_WARN("failed to exec sql", K(ret));
} else {/*do nothing*/}
if (OB_SUCC(ret)) {
if (OB_FAIL(trans.end(true))) {
LOG_WARN("fail to commit transaction", K(ret));
}
} else {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
LOG_WARN("fail to roll back transaction", K(tmp_ret));
}
}
}
return ret;
}
int ObOptStatSqlService::update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat)
{
int ret = OB_SUCCESS;
ObSqlString raw_sql;
ObSqlString value_str;
int64_t affected_rows = 0;
const uint64_t tenant_id = gen_meta_tenant_id(gather_stat.get_tenant_id());
if (OB_FAIL(get_gather_stat_value(gather_stat, value_str))) {
LOG_WARN("failed to get gather stat value", K(ret));
} else if (OB_FAIL(raw_sql.append_fmt(INSERT_TABLE_OPT_STAT_GATHER_SQL,
share::OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME,
value_str.ptr()))) {
LOG_WARN("failed to append fmt", K(ret), K(raw_sql));
} else {
ObMySQLTransaction trans;
LOG_TRACE("sql string of update opt stat gather stat", K(raw_sql));
if (OB_FAIL(trans.start(mysql_proxy_, tenant_id))) {
LOG_WARN("fail to start transaction", K(ret), K(tenant_id));
} else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) {
LOG_WARN("failed to exec sql", K(ret));
} else {/*do nothing*/}
if (OB_SUCC(ret)) {
if (OB_FAIL(trans.end(true))) {
LOG_WARN("fail to commit transaction", K(ret));
}
} else {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
LOG_WARN("fail to roll back transaction", K(tmp_ret));
}
}
}
return ret;
}
int ObOptStatSqlService::get_gather_stat_task_value(const ObOptStatTaskInfo &task_info,
ObSqlString &value_str)
{
int ret = OB_SUCCESS;
share::ObDMLSqlSplicer dml_splicer;
uint64_t tenant_id = task_info.tenant_id_;
if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id)) ||
OB_FAIL(dml_splicer.add_pk_column("task_id", task_info.task_id_)) ||
OB_FAIL(dml_splicer.add_column("type", task_info.type_)) ||
OB_FAIL(dml_splicer.add_column("ret_code", task_info.ret_code_)) ||
OB_FAIL(dml_splicer.add_column("failed_count", task_info.failed_count_)) ||
OB_FAIL(dml_splicer.add_column("table_count", task_info.task_table_count_)) ||
OB_FAIL(dml_splicer.add_time_column("start_time", task_info.task_start_time_)) ||
OB_FAIL(dml_splicer.add_time_column("end_time", task_info.task_end_time_))) {
LOG_WARN("failed to add dml splicer column", K(ret));
} else if (OB_FAIL(dml_splicer.splice_values(value_str))) {
LOG_WARN("failed to get sql string", K(ret));
} else { /*do nothing*/ }
return ret;
}
int ObOptStatSqlService::get_gather_stat_value(const ObOptStatGatherStat &gather_stat,
ObSqlString &values_ptr)
{
int ret = OB_SUCCESS;
share::ObDMLSqlSplicer dml_splicer;
uint64_t tenant_id = gather_stat.get_tenant_id();
uint64_t table_id = gather_stat.get_table_id();
uint64_t pure_table_id = ObSchemaUtils::get_extract_schema_id(tenant_id, table_id);
if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id)) ||
OB_FAIL(dml_splicer.add_pk_column("task_id", gather_stat.get_task_id())) ||
OB_FAIL(dml_splicer.add_pk_column("table_id", pure_table_id)) ||
OB_FAIL(dml_splicer.add_column("ret_code", gather_stat.get_ret_code())) ||
OB_FAIL(dml_splicer.add_time_column("start_time", gather_stat.get_start_time())) ||
OB_FAIL(dml_splicer.add_time_column("end_time", gather_stat.get_end_time())) ||
OB_FAIL(dml_splicer.add_column("memory_used", gather_stat.get_memory_used())) ||
OB_FAIL(dml_splicer.add_column("stat_refresh_failed_list", gather_stat.get_stat_refresh_failed_list())) ||
OB_FAIL(dml_splicer.add_column("properties", gather_stat.get_properties()))) {
LOG_WARN("failed to add dml splicer column", K(ret));
} else if (OB_FAIL(dml_splicer.splice_values(values_ptr))) {
LOG_WARN("failed to get sql string", K(ret));
}
return ret;
}
} // end of namespace common
} // end of namespace oceanbase