hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'column_store'

Browse Source 
Co-authored-by: wangt1xiuyi <13547954130@163.com>
Co-authored-by: yangqise7en <877793735@qq.com>
Co-authored-by: Zach41 <zach_41@163.com>
This commit is contained in:
chaser-ch
2023-10-31 15:39:20 +00:00
committed by ob-robot
parent 4057fbc4ae
commit 566e920620
1375 changed files with 239147 additions and 56014 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
src/share/stat/ob_opt_stat_sql_service.cpp
View File

@ -58,7 +58,7 @@
"sample_size," \
"bucket_cnt," \
"density," \
"last_analyzed"
"last_analyzed" \
#define INSERT_TABLE_STAT_SQL "REPLACE INTO __all_table_stat(tenant_id," \
"table_id," \
@ -99,7 +99,7 @@
"bucket_cnt," \
"histogram_type," \
"global_stats," \
"user_stats) VALUES "
"user_stats%s) VALUES "\
#define INSERT_HISTOGRAM_STAT_SQL "INSERT INTO __all_histogram_stat(tenant_id," \
@ -130,7 +130,7 @@
last_analyzed, distinct_cnt, null_cnt, max_value, b_max_value,\
min_value, b_min_value, avg_len, distinct_cnt_synopsis,\
distinct_cnt_synopsis_size, sample_size, density, bucket_cnt,\
histogram_type) VALUES"
histogram_type%s) VALUES"
#define INSERT_HISTOGRAM_STAT_HISTORY_SQL "INSERT INTO %s(tenant_id,\
table_id, partition_id, column_id, endpoint_num,savtime,\
@ -269,6 +269,36 @@
"WHERE %.*s "\
"ORDER BY tenant_id, table_id, partition_id, column_id, endpoint_num;"
#define FETCH_ALL_COLUMN_STAT_SQL_COL "SELECT col_stat.tenant_id as tenant_id, " \
"col_stat.table_id as table_id, " \
"col_stat.partition_id as partition_id, " \
"col_stat.column_id as column_id, " \
"col_stat.object_type as stat_level, " \
"col_stat.distinct_cnt as num_distinct, " \
"col_stat.null_cnt as num_null," \
"col_stat.b_max_value as b_max_value, " \
"col_stat.b_min_value as b_min_value," \
"col_stat.avg_len as avg_len," \
"col_stat.distinct_cnt_synopsis as distinct_cnt_synopsis," \
"col_stat.distinct_cnt_synopsis_size as distinct_cnt_synopsis_size," \
"col_stat.histogram_type as histogram_type," \
"col_stat.sample_size as sample_size," \
"col_stat.bucket_cnt as bucket_cnt," \
"col_stat.density as density," \
"col_stat.last_analyzed as last_analyzed,"\
"col_stat.cg_macro_blk_cnt as cg_macro_blk_cnt,"\
"col_stat.cg_micro_blk_cnt as cg_micro_blk_cnt,"\
"hist_stat.endpoint_num as endpoint_num, " \
"hist_stat.b_endpoint_value as b_endpoint_value," \
"hist_stat.endpoint_repeat_cnt as endpoint_repeat_cnt "\
"FROM %s col_stat LEFT JOIN %s hist_stat "\
"ON col_stat.tenant_id = hist_stat.tenant_id AND "\
" col_stat.table_id = hist_stat.table_id AND "\
" col_stat.partition_id = hist_stat.partition_id AND "\
" col_stat.column_id = hist_stat.column_id "\
"WHERE %.*s "\
"ORDER BY tenant_id, table_id, partition_id, column_id, endpoint_num;"
namespace oceanbase
{
@ -517,12 +547,15 @@ int ObOptStatSqlService::construct_column_stat_sql(share::schema::ObSchemaGetter
ObSqlString tmp;
ObObjMeta min_meta;
ObObjMeta max_meta;
uint64_t data_version = 0;
if (OB_FAIL(get_column_stat_min_max_meta(schema_guard, tenant_id,
is_history_stat ? share::OB_ALL_COLUMN_STAT_HISTORY_TID :
share::OB_ALL_COLUMN_STAT_TID,
min_meta,
max_meta))) {
LOG_WARN("failed to get column stat min max meta", K(ret));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) {
LOG_WARN("fail to get tenant data version", KR(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < column_stats.count(); i++) {
tmp.reset();
@ -531,7 +564,8 @@ int ObOptStatSqlService::construct_column_stat_sql(share::schema::ObSchemaGetter
LOG_WARN("column stat is null", K(ret));
} else if (is_history_stat) {
if (i == 0 && OB_FAIL(column_stats_sql.append_fmt(INSERT_COL_STAT_HISTORY_SQL,
share::OB_ALL_COLUMN_STAT_HISTORY_TNAME))) {
share::OB_ALL_COLUMN_STAT_HISTORY_TNAME,
data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt"))) {
LOG_WARN("failed to append fmt sql", K(ret));
} else if (OB_FAIL(get_column_stat_history_sql(tenant_id, allocator,
*column_stats.at(i), current_time,
@ -541,7 +575,8 @@ int ObOptStatSqlService::construct_column_stat_sql(share::schema::ObSchemaGetter
(i == column_stats.count() - 1 ? ";" : ",")))) {
LOG_WARN("failed to append sql", K(ret));
} else {/*do nothing*/}
} else if (i == 0 && OB_FAIL(column_stats_sql.append(REPLACE_COL_STAT_SQL))) {
} else if (i == 0 && OB_FAIL(column_stats_sql.append_fmt(REPLACE_COL_STAT_SQL,
data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt"))) {
LOG_WARN("failed to append sql", K(ret));
} else if (OB_FAIL(get_column_stat_sql(tenant_id, allocator,
*column_stats.at(i), current_time,
@ -901,6 +936,7 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id,
char *llc_hex_buf = NULL;
int64_t llc_comp_size = 0;
int64_t llc_hex_size = 0;
uint64_t data_version = 0;
if (OB_UNLIKELY(ObHistType::INVALID_TYPE != stat.get_histogram().get_type() &&
stat.get_histogram().get_bucket_cnt() == 0)) {
ret = OB_ERR_UNEXPECTED;
@ -911,6 +947,8 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id,
} else if (OB_FAIL(get_obj_binary_hex_str(stat.get_min_value(), allocator, b_min_str)) ||
OB_FAIL(get_obj_binary_hex_str(stat.get_max_value(), allocator, b_max_str))) {
LOG_WARN("failed to convert obj to str", K(ret));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) {
LOG_WARN("fail to get tenant data version", KR(ret));
} else if (stat.get_llc_bitmap_size() <= 0) {
// do nothing
} else if (OB_FAIL(get_compressed_llc_bitmap(allocator,
@ -951,7 +989,11 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id,
OB_FAIL(dml_splicer.add_column("bucket_cnt", stat.get_histogram().get_bucket_cnt())) ||
OB_FAIL(dml_splicer.add_column("histogram_type", stat.get_histogram().get_type())) ||
OB_FAIL(dml_splicer.add_column("global_stats", 0)) ||
OB_FAIL(dml_splicer.add_column("user_stats", 0))) {
OB_FAIL(dml_splicer.add_column("user_stats", 0)) ||
(data_version >= DATA_VERSION_4_3_0_0 &&
OB_FAIL(dml_splicer.add_column("cg_macro_blk_cnt", stat.get_cg_macro_blk_cnt()))) ||
(data_version >= DATA_VERSION_4_3_0_0 &&
OB_FAIL(dml_splicer.add_column("cg_micro_blk_cnt", stat.get_cg_micro_blk_cnt())))) {
LOG_WARN("failed to add dml splicer column", K(ret));
} else if (OB_FAIL(dml_splicer.splice_values(sql_string))) {
LOG_WARN("failed to get sql string", K(ret));
@ -980,11 +1022,14 @@ int ObOptStatSqlService::get_column_stat_history_sql(const uint64_t tenant_id,
char *llc_hex_buf = NULL;
int64_t llc_comp_size = 0;
int64_t llc_hex_size = 0;
uint64_t data_version = 0;
if (OB_FAIL(ObOptStatSqlService::get_valid_obj_str(stat.get_min_value(), min_meta, allocator, min_str, print_params)) ||
OB_FAIL(ObOptStatSqlService::get_valid_obj_str(stat.get_max_value(), max_meta, allocator, max_str, print_params)) ||
OB_FAIL(ObOptStatSqlService::get_obj_binary_hex_str(stat.get_min_value(), allocator, b_min_str)) ||
OB_FAIL(ObOptStatSqlService::get_obj_binary_hex_str(stat.get_max_value(), allocator, b_max_str))) {
LOG_WARN("failed to convert obj to str", K(ret));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) {
LOG_WARN("fail to get tenant data version", KR(ret));
} else if (stat.get_llc_bitmap_size() <= 0) {
// do nothing
} else if (OB_FAIL(ObOptStatSqlService::get_compressed_llc_bitmap(allocator,
@ -1025,7 +1070,11 @@ int ObOptStatSqlService::get_column_stat_history_sql(const uint64_t tenant_id,
OB_FAIL(dml_splicer.add_column("sample_size", stat.get_histogram().get_sample_size())) ||
OB_FAIL(dml_splicer.add_column("density", stat.get_histogram().get_density())) ||
OB_FAIL(dml_splicer.add_column("bucket_cnt", stat.get_histogram().get_bucket_cnt())) ||
OB_FAIL(dml_splicer.add_column("histogram_type", stat.get_histogram().get_type()))) {
OB_FAIL(dml_splicer.add_column("histogram_type", stat.get_histogram().get_type())) ||
(data_version >= DATA_VERSION_4_3_0_0 &&
OB_FAIL(dml_splicer.add_column("cg_macro_blk_cnt", stat.get_cg_macro_blk_cnt()))) ||
(data_version >= DATA_VERSION_4_3_0_0 &&
OB_FAIL(dml_splicer.add_column("cg_micro_blk_cnt", stat.get_cg_micro_blk_cnt())))) {
LOG_WARN("failed to add dml splicer column", K(ret));
} else if (OB_FAIL(dml_splicer.splice_values(sql_string))) {
LOG_WARN("failed to get sql string", K(ret));
@ -1271,10 +1320,19 @@ int ObOptStatSqlService::fetch_column_stat(const uint64_t tenant_id,
sqlclient::ObMySQLResult *result = NULL;
ObSqlString sql;
const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id);
uint64_t data_version = 0;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("sql service has not been initialized.", K(ret));
} else if (OB_FAIL(sql.append_fmt(FETCH_ALL_COLUMN_STAT_SQL,
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) {
LOG_WARN("fail to get tenant data version", KR(ret));
} else if (data_version < DATA_VERSION_4_3_0_0 && OB_FAIL(sql.append_fmt(FETCH_ALL_COLUMN_STAT_SQL,
share::OB_ALL_COLUMN_STAT_TNAME,
share::OB_ALL_HISTOGRAM_STAT_TNAME,
keys_list_str.string().length(),
keys_list_str.string().ptr()))) {
LOG_WARN("fail to append SQL stmt string.", K(ret));
} else if (data_version >= DATA_VERSION_4_3_0_0 && OB_FAIL(sql.append_fmt(FETCH_ALL_COLUMN_STAT_SQL_COL,
share::OB_ALL_COLUMN_STAT_TNAME,
share::OB_ALL_HISTOGRAM_STAT_TNAME,
keys_list_str.string().length(),
@ -1297,7 +1355,8 @@ int ObOptStatSqlService::fetch_column_stat(const uint64_t tenant_id,
} else if (OB_FAIL(fill_column_stat(allocator,
*result,
key_index_map,
key_col_stats))) {
key_col_stats,
data_version >= DATA_VERSION_4_3_0_0))) {
LOG_WARN("read stat from result failed. ", K(ret));
} else {/*do nothing*/}
}
@ -1310,7 +1369,8 @@ int ObOptStatSqlService::fetch_column_stat(const uint64_t tenant_id,
int ObOptStatSqlService::fill_column_stat(ObIAllocator &allocator,
common::sqlclient::ObMySQLResult &result,
hash::ObHashMap<ObOptKeyInfo, int64_t> &key_index_map,
ObIArray<ObOptKeyColumnStat> &key_col_stats)
ObIArray<ObOptKeyColumnStat> &key_col_stats,
bool need_cg_info)
{
int ret = OB_SUCCESS;
uint64_t pure_table_id = 0;
@ -1455,6 +1515,20 @@ int ObOptStatSqlService::fill_column_stat(ObIAllocator &allocator,
}
}
}
if (OB_SUCC(ret) && need_cg_info) {
EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, cg_macro_blk_cnt, *stat, int64_t, true, true, 0);
EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, cg_micro_blk_cnt, *stat, int64_t, true, true, 0);
//will be used in the future, not removed.
// if (OB_SUCC(ret)) {
// if (OB_FAIL(result.get_type("cg_skip_rate", obj_type))) {
// LOG_WARN("failed to get type", K(ret));
// } else if (OB_LIKELY(obj_type.is_double())) {
// EXTRACT_DOUBLE_FIELD_TO_CLASS_MYSQL(result, cg_skip_rate, *stat, int64_t);
// } else {
// EXTRACT_INT_FIELD_TO_CLASS_MYSQL(result, cg_skip_rate, *stat, int64_t);
// }
// }
}
}
}
}