hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix the issue of 500 tenant memory inflation caused by slice_calc in the px module

Browse Source
This commit is contained in:
obdev
2023-09-06 07:38:42 +00:00
committed by ob-robot
parent 3ddb371a82
commit f131ac541c
3 changed files with 29 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp
View File

@ -226,7 +226,7 @@ int ObPxRepartTransmitOp::do_repart_transmit(ObRepartSliceIdxCalc &repart_slice_
{
int ret = OB_SUCCESS;
// init the ObRepartSliceIdxCalc cache map
if (OB_FAIL(repart_slice_calc.init())) {
if (OB_FAIL(repart_slice_calc.init(ctx_.get_my_session()->get_effective_tenant_id()))) {
LOG_WARN("failed to init repart slice calc", K(ret));
} else if (OB_FAIL(send_rows(repart_slice_calc))) {
LOG_WARN("failed to send rows", K(ret));

36
src/sql/executor/ob_slice_calc.cpp
View File

@ -237,10 +237,10 @@ int ObRepartSliceIdxCalc::get_previous_row_tablet_id(ObObj &tablet_id)
return ret;
}
int ObSlaveMapRepartIdxCalcBase::init()
int ObSlaveMapRepartIdxCalcBase::init(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObRepartSliceIdxCalc::init())) {
if (OB_FAIL(ObRepartSliceIdxCalc::init(tenant_id))) {
LOG_WARN("fail init base", K(ret));
}
// 在pkey random情况下，一个partition是可以被其所在的SQC上的所有worker处理的，
@ -254,7 +254,10 @@ int ObSlaveMapRepartIdxCalcBase::init()
// p2 : [task4,task5]
const ObPxPartChMapTMArray &part_ch_array = part_ch_info_.part_ch_array_;
if (OB_SUCC(ret)) {
if (OB_FAIL(part_to_task_array_map_.create(max(1, part_ch_array.count()), ObModIds::OB_SQL_PX))) {
if (OB_FAIL(part_to_task_array_map_.create(max(1, part_ch_array.count()),
ObModIds::OB_SQL_PX,
ObModIds::OB_HASH_NODE,
tenant_id))) {
LOG_WARN("fail create part to task array map", "count", part_ch_array.count(), K(ret));
} else {
// In ObRepartSliceIdxCalc::init(), the support_vectorized_calc_ has been set to true.
@ -304,9 +307,9 @@ int ObSlaveMapRepartIdxCalcBase::destroy()
return ret;
}
int ObRepartRandomSliceIdxCalc::init()
int ObRepartRandomSliceIdxCalc::init(uint64_t tenant_id)
{
return ObSlaveMapRepartIdxCalcBase::init();
return ObSlaveMapRepartIdxCalcBase::init(tenant_id);
}
int ObRepartRandomSliceIdxCalc::destroy()
@ -471,13 +474,13 @@ int ObAffinitizedRepartSliceIdxCalc::get_slice_idx_vec(const ObIArray<ObExpr*> &
return ret;
}
int ObRepartSliceIdxCalc::init()
int ObRepartSliceIdxCalc::init(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (px_repart_ch_map_.created()) {
ret = OB_INIT_TWICE;
LOG_WARN("this map has been init twice", K(ret));
} else if (OB_FAIL(build_repart_ch_map(px_repart_ch_map_))) {
} else if (OB_FAIL(build_repart_ch_map(px_repart_ch_map_, tenant_id))) {
LOG_WARN("failed to build affi hash map", K(ret));
} else if (OB_FAIL(setup_one_side_one_level_info())) {
LOG_WARN("fail to build one side on level map", K(ret));
@ -515,11 +518,14 @@ int ObRepartSliceIdxCalc::setup_one_side_one_level_info()
return ret;
}
int ObRepartSliceIdxCalc::build_repart_ch_map(ObPxPartChMap &affinity_map)
int ObRepartSliceIdxCalc::build_repart_ch_map(ObPxPartChMap &affinity_map, uint64_t tenant_id)
{
int ret = OB_SUCCESS;
const ObPxPartChMapTMArray &part_ch_array = part_ch_info_.part_ch_array_;
if (OB_FAIL(affinity_map.create(max(1, part_ch_array.count()), ObModIds::OB_SQL_PX))) {
if (OB_FAIL(affinity_map.create(max(1, part_ch_array.count()),
ObModIds::OB_SQL_PX,
ObModIds::OB_HASH_NODE,
tenant_id))) {
LOG_WARN("fail create hashmap", "count", part_ch_array.count(), K(ret));
}
@ -795,13 +801,13 @@ ObSlaveMapPkeyRangeIdxCalc::~ObSlaveMapPkeyRangeIdxCalc()
destroy();
}
int ObSlaveMapPkeyRangeIdxCalc::init()
int ObSlaveMapPkeyRangeIdxCalc::init(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret), K(is_inited_));
} else if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init())) {
} else if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init(tenant_id))) {
LOG_WARN("fail init base repart class", K(ret));
} else if (OB_UNLIKELY(nullptr == calc_part_id_expr_ || sort_exprs_.count() <= 0)) {
ret = OB_INVALID_ARGUMENT;
@ -1048,10 +1054,10 @@ int ObSlaveMapPkeyRangeIdxCalc::get_slice_idx(
}
/******************* ObSlaveMapPkeyHashIdxCalc ********************/
int ObSlaveMapPkeyHashIdxCalc::init()
int ObSlaveMapPkeyHashIdxCalc::init(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init())) {
if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init(tenant_id))) {
LOG_WARN("fail init base repart class", K(ret));
} else if (affi_hash_map_.created()) {
ret = OB_INIT_TWICE;
@ -1339,10 +1345,10 @@ int ObNullAwareHashSliceIdCalc::get_slice_indexes(const ObIArray<ObExpr*> &exprs
return ret;
}
int ObNullAwareAffinitizedRepartSliceIdxCalc::init()
int ObNullAwareAffinitizedRepartSliceIdxCalc::init(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
OZ (ObRepartSliceIdxCalc::init());
OZ (ObRepartSliceIdxCalc::init(tenant_id));
OX (support_vectorized_calc_ = false);
return ret;
}

14
src/sql/executor/ob_slice_calc.h
View File

@ -232,7 +232,7 @@ public:
int init_partition_cache_map();
virtual int init();
virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID);
virtual int destroy() {
int ret = OB_SUCCESS;
@ -242,7 +242,7 @@ public:
return ret;
}
int build_repart_ch_map(ObPxPartChMap &map);
int build_repart_ch_map(ObPxPartChMap &map, uint64_t tenant_id);
int build_part2tablet_id_map();
protected:
// this is a trick!
@ -320,7 +320,7 @@ protected:
{}
~ObSlaveMapRepartIdxCalcBase() = default;
protected:
virtual int init() override;
virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override;
virtual int destroy() override;
protected:
// 存储同一个partition所对应的所有task id
@ -390,7 +390,7 @@ public:
virtual int get_slice_idx(const ObIArray<ObExpr*> &exprs,
ObEvalCtx &eval_ctx,
int64_t &slice_idx) override;
virtual int init() override;
virtual int init(uint64_t tenant_id) override;
virtual int destroy() override;
private:
int get_task_idx_by_tablet_id(int64_t tablet_id , int64_t &task_idx);
@ -810,7 +810,7 @@ public:
sort_cmp_(sort_cmp_funs, sort_collations)
{}
virtual ~ObSlaveMapPkeyRangeIdxCalc();
virtual int init() override;
virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override;
virtual int destroy() override;
virtual int get_slice_idx(
const common::ObIArray<ObExpr*> &exprs,
@ -932,7 +932,7 @@ public:
}
~ObSlaveMapPkeyHashIdxCalc() = default;
int init() override;
int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override;
int destroy() override;
// for static engine
@ -1091,7 +1091,7 @@ public:
}
~ObNullAwareAffinitizedRepartSliceIdxCalc() = default;
virtual int init() override;
virtual int init(uint64_t tenant_id) override;
virtual int get_slice_idx(const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx)
{
UNUSED(exprs);