hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] performance optimzation for OLTP

Browse Source 
Co-authored-by: dimstars <liangjinrongcm@gmail.com>
Co-authored-by: pe-99y <315053752@qq.com>
This commit is contained in:
Naynahs
2024-04-10 07:32:27 +00:00
committed by ob-robot
parent 054f5a5a80
commit 3d4ef9741d
177 changed files with 7111 additions and 9708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/sql/plan_cache/ob_dist_plans.cpp
View File

@ -69,7 +69,6 @@ int ObDistPlans::get_plan(ObPlanCacheCtx &pc_ctx,
// for single dist plan without px, we already fill the phy locations while calculating plan type
// for multi table px plan, physical location is calculated in match step
ObArray<ObCandiTableLoc> candi_table_locs;
bool need_check_on_same_server = false;
if (OB_ISNULL(plan_set_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid null plan set", K(ret), K(plan_set_));
@ -77,12 +76,13 @@ int ObDistPlans::get_plan(ObPlanCacheCtx &pc_ctx,
// do nothing
} else if (OB_FAIL(ObPhyLocationGetter::get_phy_locations(plan->get_table_locations(),
pc_ctx,
candi_table_locs,
need_check_on_same_server))) {
candi_table_locs))) {
LOG_WARN("failed to get physical table locations", K(ret));
} else if (OB_FAIL(ObPhyLocationGetter::build_table_locs(pc_ctx.exec_ctx_.get_das_ctx(),
plan->get_table_locations(),
candi_table_locs))) {
} else if (candi_table_locs.empty()) {
// do nothing.
} else if (OB_FAIL(ObPhyLocationGetter::build_candi_table_locs(pc_ctx.exec_ctx_.get_das_ctx(),
plan->get_table_locations(),
candi_table_locs))) {
LOG_WARN("fail to init table locs", K(ret));
}
}

4
src/sql/plan_cache/ob_pcv_set.cpp
View File

@ -140,7 +140,7 @@ int ObPCVSet::inner_get_cache_obj(ObILibCacheCtx &ctx,
}
}
if (pc_ctx.exec_ctx_.get_min_cluster_version() != GET_MIN_CLUSTER_VERSION()) {
LOG_TRACE("Lob Debug, using remote min cluster version",
LOG_DEBUG("Lob Debug, using remote min cluster version",
K(pc_ctx.exec_ctx_.get_min_cluster_version()),
K(GET_MIN_CLUSTER_VERSION()));
}
@ -181,7 +181,7 @@ int ObPCVSet::inner_get_cache_obj(ObILibCacheCtx &ctx,
bool need_check_schema = true;
DLIST_FOREACH(pcv, pcv_list_) {
bool is_same = false;
LOG_TRACE("get plan, pcv", K(pcv));
LOG_DEBUG("get plan, pcv", K(pcv));
if (OB_FAIL(pcv->get_all_dep_schema(pc_ctx,
pc_ctx.sql_ctx_.session_info_->get_database_id(),
new_tenant_schema_version,

10
src/sql/plan_cache/ob_plan_cache.cpp
View File

@ -740,8 +740,6 @@ int ObPlanCache::construct_fast_parser_result(common::ObIAllocator &allocator,
K(batch_count), K(first_truncated_sql), K(pc_ctx.raw_sql_), K(fp_result));
} else {
fp_result.raw_params_.reset();
fp_result.raw_params_.set_allocator(&allocator);
fp_result.raw_params_.set_capacity(pc_ctx.insert_batch_opt_info_.multi_raw_params_.at(0)->count());
if (OB_FAIL(fp_result.raw_params_.assign(*pc_ctx.insert_batch_opt_info_.multi_raw_params_.at(0)))) {
LOG_WARN("fail to assign raw_param", K(ret));
} else {
@ -1229,18 +1227,18 @@ int ObPlanCache::get_cache_obj(ObILibCacheCtx &ctx,
SQL_PC_LOG(TRACE, "failed to get cache node from lib cache by key", K(ret));
} else if (OB_UNLIKELY(NULL == cache_node)) {
ret = OB_SQL_PC_NOT_EXIST;
SQL_PC_LOG(TRACE, "cache obj does not exist!", K(key));
SQL_PC_LOG(DEBUG, "cache obj does not exist!", K(key));
} else {
LOG_TRACE("inner_get_cache_obj", K(key), K(cache_node));
LOG_DEBUG("inner_get_cache_obj", K(key), K(cache_node));
if (OB_FAIL(cache_node->update_node_stat(ctx))) {
SQL_PC_LOG(WARN, "failed to update node stat", K(ret));
} else if (OB_FAIL(cache_node->get_cache_obj(ctx, key, cache_obj))) {
if (OB_SQL_PC_NOT_EXIST != ret) {
LOG_TRACE("cache_node fail to get cache obj", K(ret));
LOG_DEBUG("cache_node fail to get cache obj", K(ret));
}
} else {
guard.cache_obj_ = cache_obj;
LOG_TRACE("succ to get cache obj", KPC(key));
LOG_DEBUG("succ to get cache obj", KPC(key));
}
// release lock whatever
(void)cache_node->unlock();

4
src/sql/plan_cache/ob_plan_cache_struct.h
View File

@ -222,7 +222,6 @@ private:
public:
ObFastParserResult()
: inner_alloc_("FastParserRes"),
raw_params_(&inner_alloc_),
parameterized_params_(&inner_alloc_),
cache_params_(NULL),
values_token_pos_(0),
@ -231,7 +230,7 @@ public:
reset_question_mark_ctx();
}
ObPlanCacheKey pc_key_; //plan cache key, parameterized by fast parser
common::ObFixedArray<ObPCParam *, common::ObIAllocator> raw_params_;
common::ObSEArray<ObPCParam *, 4> raw_params_;
common::ObFixedArray<const common::ObObjParam *, common::ObIAllocator> parameterized_params_;
ParamStore *cache_params_;
ObQuestionMarkCtx question_mark_ctx_;
@ -261,7 +260,6 @@ public:
{
int ret = OB_SUCCESS;
pc_key_ = other.pc_key_;
raw_params_.set_allocator(&inner_alloc_);
parameterized_params_.set_allocator(&inner_alloc_);
cache_params_ = other.cache_params_;
question_mark_ctx_ = other.question_mark_ctx_;

123
src/sql/plan_cache/ob_plan_cache_util.cpp
View File

@ -251,7 +251,6 @@ int ObPhyLocationGetter::reselect_duplicate_table_best_replica(const ObIArray<Ob
return ret;
}
//need_check_on_same_server: out, 是否需要检查分区在同一server, 如果这里检查过了就置为false
int ObPhyLocationGetter::get_phy_locations(const ObIArray<ObTableLocation> &table_locations,
const ObPlanCacheCtx &pc_ctx,
ObIArray<ObCandiTableLoc> &candi_table_locs,
@ -336,6 +335,106 @@ int ObPhyLocationGetter::get_phy_locations(const ObIArray<ObTableLocation> &tabl
return ret;
}
int ObPhyLocationGetter::get_phy_locations(const ObIArray<ObTableLocation> &table_locations,
const ObPlanCacheCtx &pc_ctx,
ObIArray<ObCandiTableLoc> &candi_table_locs)
{
int ret = OB_SUCCESS;
bool has_duplicate_tbl_not_in_dml = false;
ObExecContext &exec_ctx = pc_ctx.exec_ctx_;
const ObDataTypeCastParams dtc_params = ObBasicSessionInfo::create_dtc_params(pc_ctx.sql_ctx_.session_info_);
ObPhysicalPlanCtx *plan_ctx = exec_ctx.get_physical_plan_ctx();
const ParamStore &params = plan_ctx->get_param_store();
int64_t N = table_locations.count();
bool is_retrying = false;
if (OB_ISNULL(plan_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid executor ctx!", K(ret), K(plan_ctx));
} else {
int64_t calculate_candi_table_num = N;
for (int64_t i = 0; OB_SUCC(ret) && i < N ; i++) {
const ObTableLocation &table_location = table_locations.at(i);
if (table_location.get_loc_meta().select_leader_) {
if (OB_FAIL(table_location.calculate_final_tablet_locations(exec_ctx,
params,
dtc_params))) {
LOG_WARN("failed to calculate final tablet locations", K(table_location), K(ret));
} else {
calculate_candi_table_num -= 1;
}
}
} // for end
if (OB_SUCC(ret) && calculate_candi_table_num > 0) {
ObSEArray<const ObTableLocation *, 2> table_location_ptrs;
ObSEArray<ObCandiTableLoc *, 2> phy_location_info_ptrs;
if (OB_FAIL(candi_table_locs.prepare_allocate(calculate_candi_table_num))) {
LOG_WARN("phy_locations_info prepare allocate error", K(ret), K(calculate_candi_table_num));
} else {
for (int64_t i = 0, j = 0; OB_SUCC(ret) && i < N && j < calculate_candi_table_num; i++) {
const ObTableLocation &table_location = table_locations.at(i);
if (!table_location.get_loc_meta().select_leader_) {
ObCandiTableLoc &candi_table_loc = candi_table_locs.at(j);
NG_TRACE(calc_partition_location_begin);
// 这里认为materialized view的复制表是每个server都有副本的，
// 因此这里不判断是否能生成materialized view了，一定都能生成
if (OB_FAIL(table_location.calculate_candi_tablet_locations(exec_ctx,
params,
candi_table_loc.get_phy_part_loc_info_list_for_update(),
dtc_params))) {
LOG_WARN("failed to calculate partition location", K(ret));
} else {
NG_TRACE(calc_partition_location_end);
if (table_location.is_duplicate_table_not_in_dml()) {
has_duplicate_tbl_not_in_dml = true;
}
candi_table_loc.set_duplicate_type(table_location.get_duplicate_type());
candi_table_loc.set_table_location_key(
table_location.get_table_id(), table_location.get_ref_table_id());
LOG_DEBUG("plan cache util", K(candi_table_loc));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(table_location_ptrs.push_back(&table_location))) {
LOG_WARN("failed to push back table location ptrs", K(ret), K(i),
K(N), K(table_locations.at(i)));
} else if (OB_FAIL(phy_location_info_ptrs.push_back(&candi_table_loc))) {
LOG_WARN("failed to push back phy location info ptrs", K(ret), K(i),
K(N), K(candi_table_locs.at(j)));
} else if (OB_FAIL(pc_ctx.is_retry_for_dup_tbl(is_retrying))) {
LOG_WARN("failed to test if retrying", K(ret));
} else if (is_retrying) {
LOG_INFO("Physical Location from Location Cache", K(candi_table_loc));
}
j += 1;
}
}
} // for end
}
//Only check the on_same_server when has table location in the phy_plan.
if (OB_SUCC(ret)) {
bool on_same_server = true;
if (OB_FAIL(ObLogPlan::select_replicas(exec_ctx, table_location_ptrs,
exec_ctx.get_addr(),
phy_location_info_ptrs))) {
LOG_WARN("failed to select replicas", K(ret), K(table_locations),
K(exec_ctx.get_addr()), K(phy_location_info_ptrs));
} else if (!has_duplicate_tbl_not_in_dml || is_retrying) {
// do nothing
} else if (OB_FAIL(reselect_duplicate_table_best_replica(candi_table_locs,
on_same_server))) {
LOG_WARN("failed to reselect replicas", K(ret));
}
LOG_TRACE("after select_replicas", K(on_same_server), K(has_duplicate_tbl_not_in_dml),
K(candi_table_locs), K(table_locations), K(ret));
}
}
}
return ret;
}
int ObPhyLocationGetter::build_table_locs(ObDASCtx &das_ctx,
const ObIArray<ObTableLocation> &table_locations,
const ObIArray<ObCandiTableLoc> &candi_table_locs)
@ -354,6 +453,28 @@ int ObPhyLocationGetter::build_table_locs(ObDASCtx &das_ctx,
return ret;
}
int ObPhyLocationGetter::build_candi_table_locs(ObDASCtx &das_ctx,
const ObIArray<ObTableLocation> &table_locations,
const ObIArray<ObCandiTableLoc> &candi_table_locs)
{
int ret = OB_SUCCESS;
CK(table_locations.count() >= candi_table_locs.count());
for (int64_t i = 0, j = 0; OB_SUCC(ret) && i < table_locations.count() && j < candi_table_locs.count(); i++) {
if (!table_locations.at(i).get_loc_meta().select_leader_) {
if (OB_FAIL(das_ctx.add_candi_table_loc(table_locations.at(i).get_loc_meta(), candi_table_locs.at(j)))) {
LOG_WARN("add candi table location failed", K(ret), K(table_locations.at(i).get_loc_meta()));
} else {
j += 1;
}
}
}
if (OB_FAIL(ret)) {
das_ctx.clear_all_location_info();
}
return ret;
}
//this function will rewrite the related tablet map info in DASCtx
int ObPhyLocationGetter::build_related_tablet_info(const ObTableLocation &table_location,
ObExecContext &exec_ctx,

15
src/sql/plan_cache/ob_plan_cache_util.h
View File

@ -953,9 +953,19 @@ struct ObPhyLocationGetter
{
public:
// used for getting plan
// In this interface, we first process the table locations that were marked select_leader, the tablet
// locations of them will be added to das_ctx directly, without the need to construct candi_table_locs.
// For the remaining table locations that are not marked select_leader, continue to use the previous
// logic where a candi_table_loc is generated for each table location. These candi_table_locs will be
// added to das_ctx by @build_candi_table_locs().
static int get_phy_locations(const ObIArray<ObTableLocation> &table_locations,
const ObPlanCacheCtx &pc_ctx,
ObIArray<ObCandiTableLoc> &phy_location_infos,
ObIArray<ObCandiTableLoc> &phy_location_infos);
// used for matching plan
static int get_phy_locations(const ObIArray<ObTableLocation> &table_locations,
const ObPlanCacheCtx &pc_ctx,
ObIArray<ObCandiTableLoc> &candi_table_locs,
bool &need_check_on_same_server);
// used for adding plan
@ -966,6 +976,9 @@ public:
static int build_table_locs(ObDASCtx &das_ctx,
const common::ObIArray<ObTableLocation> &table_locations,
const common::ObIArray<ObCandiTableLoc> &candi_table_locs);
static int build_candi_table_locs(ObDASCtx &das_ctx,
const common::ObIArray<ObTableLocation> &table_locations,
const common::ObIArray<ObCandiTableLoc> &candi_table_locs);
static int build_related_tablet_info(const ObTableLocation &table_location,
ObExecContext &exec_ctx,
DASRelatedTabletMap *&related_map);

5
src/sql/plan_cache/ob_plan_match_helper.cpp
View File

@ -112,11 +112,6 @@ int ObPlanMatchHelper::match_plan(const ObPlanCacheCtx &pc_ctx,
LOG_WARN("failed to set refactored", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(das_ctx.add_candi_table_loc(out_tbl_locations.at(i).get_loc_meta(), src_location))) {
LOG_WARN("add candi table loc failed", K(ret), K(out_tbl_locations.at(i).get_loc_meta()), K(src_location));
}
}
}
}
}

73
src/sql/plan_cache/ob_plan_set.cpp
View File

@ -2048,22 +2048,21 @@ void ObSqlPlanSet::reset()
//need_check_on_same_server: out, 是否需要检查分区在同一server, 如果里面检查过且不在同一server则置为false
int ObSqlPlanSet::get_phy_locations(const ObIArray<ObTableLocation> &table_locations,
ObPlanCacheCtx &pc_ctx,
ObIArray<ObCandiTableLoc> &candi_table_locs,
bool &need_check_on_same_server)
ObIArray<ObCandiTableLoc> &candi_table_locs)
{
int ret = OB_SUCCESS;
need_check_on_same_server = true;
DAS_CTX(pc_ctx.exec_ctx_).clear_all_location_info();
if (OB_FAIL(ObPhyLocationGetter::get_phy_locations(table_locations,
pc_ctx,
candi_table_locs,
need_check_on_same_server))) {
candi_table_locs))) {
LOG_WARN("failed to get phy locations", K(ret), K(table_locations));
} else if (OB_FAIL(ObPhyLocationGetter::build_table_locs(pc_ctx.exec_ctx_.get_das_ctx(),
table_locations,
candi_table_locs))) {
LOG_WARN("fail to init table locs", K(ret));
}
} else if (candi_table_locs.empty()) {
// do nothing.
} else if (OB_FAIL(ObPhyLocationGetter::build_candi_table_locs(pc_ctx.exec_ctx_.get_das_ctx(),
table_locations,
candi_table_locs))) {
LOG_WARN("fail to init table locs", K(ret));
}
return ret;
}
@ -2079,57 +2078,47 @@ int ObSqlPlanSet::get_phy_locations(const ObIArray<ObTableLocation> &table_locat
* FALSE: we know partitions on different servers via ObPhyLocationGetter::get_phy_locations
* (when there are duplicate tables not in DML), no need to check again
*/
int ObSqlPlanSet::calc_phy_plan_type_v2(const ObIArray<ObCandiTableLoc> &candi_table_locs,
ObPhyPlanType &plan_type,
bool need_check_on_same_server)
int ObSqlPlanSet::calc_phy_plan_type_v2(const common::ObIArray<ObCandiTableLoc> &candi_table_locs,
const ObPlanCacheCtx &pc_ctx,
ObPhyPlanType &plan_type)
{
int ret = OB_SUCCESS;
int64_t N = candi_table_locs.count();
ObDASCtx &das_ctx = pc_ctx.exec_ctx_.get_das_ctx();
const DASTableLocList &table_locs = das_ctx.get_table_loc_list();
int64_t N = table_locs.size();
if (0 == N) {
plan_type = OB_PHY_PLAN_LOCAL;
SQL_PC_LOG(DEBUG, "no table used, thus local plan");
} else {
bool is_all_empty = true;
bool is_all_single_partition = true;
for (int i = 0; is_all_single_partition && i < N; ++i) {
if (candi_table_locs.at(i).get_partition_cnt() != 0) {
FOREACH_X(table_loc, table_locs, is_all_single_partition)
{
const DASTabletLocList &tablet_locs = (*table_loc)->get_tablet_locs();
if (tablet_locs.size() != 0) {
is_all_empty = false;
}
if (candi_table_locs.at(i).get_partition_cnt() > 1) {
if (tablet_locs.size() > 1) {
is_all_single_partition = false;
}
}
if (is_all_empty) {
plan_type = OB_PHY_PLAN_LOCAL;
} else if (is_all_single_partition) {
bool is_same = true;
ObAddr my_address = GCTX.self_addr();
ObAddr first_addr;
if (!need_check_on_same_server) {
is_same = false;
}
if (is_same && OB_FAIL(is_partition_in_same_server(candi_table_locs,
is_same,
first_addr))) {
SQL_PC_LOG(WARN, "fail to calculate whether all partitions in same server",
K(ret),
K(candi_table_locs));
} else {
if (is_same) {
if (my_address == first_addr) {
plan_type = OB_PHY_PLAN_LOCAL;
} else {
plan_type = OB_PHY_PLAN_REMOTE;
}
if (das_ctx.same_server_) {
if (GCTX.self_addr() == das_ctx.same_tablet_addr()) {
plan_type = OB_PHY_PLAN_LOCAL;
} else {
plan_type = OB_PHY_PLAN_DISTRIBUTED;
plan_type = OB_PHY_PLAN_REMOTE;
}
} else {
plan_type = OB_PHY_PLAN_DISTRIBUTED;
}
} else {
plan_type = OB_PHY_PLAN_DISTRIBUTED;
}
}
return ret;
}
@ -2347,17 +2336,15 @@ int ObSqlPlanSet::get_plan_type(const ObIArray<ObTableLocation> &table_locations
ObPhyPlanType &plan_type)
{
int ret = OB_SUCCESS;
bool need_check_on_same_server = true;
candi_table_locs.reuse();
if (OB_FAIL(get_phy_locations(table_locations,
pc_ctx,
candi_table_locs,
need_check_on_same_server))) {
candi_table_locs))) {
LOG_WARN("failed to get physical locations", K(ret));
} else if (OB_FAIL(calc_phy_plan_type_v2(candi_table_locs,
plan_type,
need_check_on_same_server))) {
pc_ctx,
plan_type))) {
LOG_WARN("failed to calcute physical plan type", K(ret));
} else {
// Lookup算子支持压到远程去执行:

9
src/sql/plan_cache/ob_plan_set.h
View File

@ -349,9 +349,9 @@ public:
// calculate phy_plan type:
// @param [in] phy_locations
// @param [out] plan_type
static int calc_phy_plan_type_v2(const common::ObIArray<ObCandiTableLoc> &phy_locations,
ObPhyPlanType &plan_type,
bool need_check_on_same_server);
static int calc_phy_plan_type_v2(const common::ObIArray<ObCandiTableLoc> &candi_table_locs,
const ObPlanCacheCtx &pc_ctx,
ObPhyPlanType &plan_type);
static int calc_phy_plan_type(const common::ObIArray<ObCandiTableLoc> &phy_locations,
ObPhyPlanType &plan_type);
inline bool has_duplicate_table() const { return has_duplicate_table_; }
@ -411,8 +411,7 @@ private:
ObPhysicalPlan *&plan);
int get_phy_locations(const ObIArray<ObTableLocation> &table_locations,
ObPlanCacheCtx &pc_ctx,
ObIArray<ObCandiTableLoc> &candi_table_locs,
bool &need_check_on_same_server);
ObIArray<ObCandiTableLoc> &candi_table_locs);
int get_phy_locations(const ObTablePartitionInfoArray &partition_infos,
ObIArray<ObCandiTableLoc> &candi_table_locs);

4
src/sql/plan_cache/ob_sql_parameterization.cpp
View File

@ -1502,8 +1502,6 @@ int ObSqlParameterization::fast_parser(ObIAllocator &allocator,
ObPCParam *pc_param = NULL;
char *ptr = (char *)allocator.alloc(param_num * sizeof(ObPCParam));
fp_result.raw_params_.reset();
fp_result.raw_params_.set_allocator(&allocator);
fp_result.raw_params_.set_capacity(param_num);
if (OB_ISNULL(ptr)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
SQL_PC_LOG(WARN, "fail to alloc memory for pc param", K(ret), K(ptr));
@ -1533,8 +1531,6 @@ int ObSqlParameterization::fast_parser(ObIAllocator &allocator,
ObPCParam *pc_param = NULL;
ParamList *p_list = parse_result.param_nodes_;
char *ptr = (char *)allocator.alloc(param_num * sizeof(ObPCParam));
fp_result.raw_params_.set_allocator(&allocator);
fp_result.raw_params_.set_capacity(param_num);
if (OB_ISNULL(ptr)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
SQL_PC_LOG(WARN, "fail to alloc memory for pc param", K(ret), K(ptr));

6
src/sql/plan_cache/ob_values_table_compression.cpp
View File

@ -12,8 +12,6 @@
#define USING_LOG_PREFIX SQL_PC
#include "sql/plan_cache/ob_values_table_compression.h"
#include "sql/plan_cache/ob_plan_cache_struct.h"
#include "sql/parser/ob_fast_parser.h"
#include "sql/engine/ob_exec_context.h"
#include "sql/resolver/ob_resolver_utils.h"
#include "sql/engine/expr/ob_expr_version.h"
@ -382,8 +380,6 @@ int ObValuesTableCompression::try_batch_exec_params(ObIAllocator &allocator,
} else if (can_fold_params) {
fp_result.pc_key_.name_.assign_ptr(new_no_param_sql.ptr(), new_no_param_sql.length());
fp_result.raw_params_.reset();
fp_result.raw_params_.set_allocator(&allocator);
fp_result.raw_params_.set_capacity(temp_store.count());
for (int64_t i = 0; i < temp_store.count(); i++) {
// checked null before
temp_store.at(i)->node_->pos_ = no_param_pos.at(i);
@ -866,4 +862,4 @@ int ObValuesTableCompression::parser_values_row_str(ObIAllocator &allocator,
}
}
}
}

2
src/sql/plan_cache/ob_values_table_compression.h
View File

@ -95,4 +95,4 @@ private:
}
}
#endif /* _OB_VALUES_TABLE_COMPRESSION_H */
#endif /* _OB_VALUES_TABLE_COMPRESSION_H */