From f131ac541c2b78f9c094662df4a0718ef2821762 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 6 Sep 2023 07:38:42 +0000 Subject: [PATCH] Fix the issue of 500 tenant memory inflation caused by slice_calc in the px module --- .../px/exchange/ob_px_repart_transmit_op.cpp | 2 +- src/sql/executor/ob_slice_calc.cpp | 36 +++++++++++-------- src/sql/executor/ob_slice_calc.h | 14 ++++---- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp b/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp index d79cd93654..fee0935a73 100644 --- a/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp @@ -226,7 +226,7 @@ int ObPxRepartTransmitOp::do_repart_transmit(ObRepartSliceIdxCalc &repart_slice_ { int ret = OB_SUCCESS; // init the ObRepartSliceIdxCalc cache map - if (OB_FAIL(repart_slice_calc.init())) { + if (OB_FAIL(repart_slice_calc.init(ctx_.get_my_session()->get_effective_tenant_id()))) { LOG_WARN("failed to init repart slice calc", K(ret)); } else if (OB_FAIL(send_rows(repart_slice_calc))) { LOG_WARN("failed to send rows", K(ret)); diff --git a/src/sql/executor/ob_slice_calc.cpp b/src/sql/executor/ob_slice_calc.cpp index ad98e3bdf7..d857c217c7 100644 --- a/src/sql/executor/ob_slice_calc.cpp +++ b/src/sql/executor/ob_slice_calc.cpp @@ -237,10 +237,10 @@ int ObRepartSliceIdxCalc::get_previous_row_tablet_id(ObObj &tablet_id) return ret; } -int ObSlaveMapRepartIdxCalcBase::init() +int ObSlaveMapRepartIdxCalcBase::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - if (OB_FAIL(ObRepartSliceIdxCalc::init())) { + if (OB_FAIL(ObRepartSliceIdxCalc::init(tenant_id))) { LOG_WARN("fail init base", K(ret)); } // 在pkey random情况下，一个partition是可以被其所在的SQC上的所有worker处理的， @@ -254,7 +254,10 @@ int ObSlaveMapRepartIdxCalcBase::init() // p2 : [task4,task5] const ObPxPartChMapTMArray &part_ch_array = part_ch_info_.part_ch_array_; if (OB_SUCC(ret)) { - if (OB_FAIL(part_to_task_array_map_.create(max(1, part_ch_array.count()), ObModIds::OB_SQL_PX))) { + if (OB_FAIL(part_to_task_array_map_.create(max(1, part_ch_array.count()), + ObModIds::OB_SQL_PX, + ObModIds::OB_HASH_NODE, + tenant_id))) { LOG_WARN("fail create part to task array map", "count", part_ch_array.count(), K(ret)); } else { // In ObRepartSliceIdxCalc::init(), the support_vectorized_calc_ has been set to true. @@ -304,9 +307,9 @@ int ObSlaveMapRepartIdxCalcBase::destroy() return ret; } -int ObRepartRandomSliceIdxCalc::init() +int ObRepartRandomSliceIdxCalc::init(uint64_t tenant_id) { - return ObSlaveMapRepartIdxCalcBase::init(); + return ObSlaveMapRepartIdxCalcBase::init(tenant_id); } int ObRepartRandomSliceIdxCalc::destroy() @@ -471,13 +474,13 @@ int ObAffinitizedRepartSliceIdxCalc::get_slice_idx_vec(const ObIArray & return ret; } -int ObRepartSliceIdxCalc::init() +int ObRepartSliceIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; if (px_repart_ch_map_.created()) { ret = OB_INIT_TWICE; LOG_WARN("this map has been init twice", K(ret)); - } else if (OB_FAIL(build_repart_ch_map(px_repart_ch_map_))) { + } else if (OB_FAIL(build_repart_ch_map(px_repart_ch_map_, tenant_id))) { LOG_WARN("failed to build affi hash map", K(ret)); } else if (OB_FAIL(setup_one_side_one_level_info())) { LOG_WARN("fail to build one side on level map", K(ret)); @@ -515,11 +518,14 @@ int ObRepartSliceIdxCalc::setup_one_side_one_level_info() return ret; } -int ObRepartSliceIdxCalc::build_repart_ch_map(ObPxPartChMap &affinity_map) +int ObRepartSliceIdxCalc::build_repart_ch_map(ObPxPartChMap &affinity_map, uint64_t tenant_id) { int ret = OB_SUCCESS; const ObPxPartChMapTMArray &part_ch_array = part_ch_info_.part_ch_array_; - if (OB_FAIL(affinity_map.create(max(1, part_ch_array.count()), ObModIds::OB_SQL_PX))) { + if (OB_FAIL(affinity_map.create(max(1, part_ch_array.count()), + ObModIds::OB_SQL_PX, + ObModIds::OB_HASH_NODE, + tenant_id))) { LOG_WARN("fail create hashmap", "count", part_ch_array.count(), K(ret)); } @@ -795,13 +801,13 @@ ObSlaveMapPkeyRangeIdxCalc::~ObSlaveMapPkeyRangeIdxCalc() destroy(); } -int ObSlaveMapPkeyRangeIdxCalc::init() +int ObSlaveMapPkeyRangeIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(is_inited_)); - } else if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init())) { + } else if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init(tenant_id))) { LOG_WARN("fail init base repart class", K(ret)); } else if (OB_UNLIKELY(nullptr == calc_part_id_expr_ || sort_exprs_.count() <= 0)) { ret = OB_INVALID_ARGUMENT; @@ -1048,10 +1054,10 @@ int ObSlaveMapPkeyRangeIdxCalc::get_slice_idx( } /******************* ObSlaveMapPkeyHashIdxCalc ********************/ -int ObSlaveMapPkeyHashIdxCalc::init() +int ObSlaveMapPkeyHashIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init())) { + if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init(tenant_id))) { LOG_WARN("fail init base repart class", K(ret)); } else if (affi_hash_map_.created()) { ret = OB_INIT_TWICE; @@ -1339,10 +1345,10 @@ int ObNullAwareHashSliceIdCalc::get_slice_indexes(const ObIArray &exprs return ret; } -int ObNullAwareAffinitizedRepartSliceIdxCalc::init() +int ObNullAwareAffinitizedRepartSliceIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - OZ (ObRepartSliceIdxCalc::init()); + OZ (ObRepartSliceIdxCalc::init(tenant_id)); OX (support_vectorized_calc_ = false); return ret; } diff --git a/src/sql/executor/ob_slice_calc.h b/src/sql/executor/ob_slice_calc.h index 45f1baa17f..81026da4d8 100644 --- a/src/sql/executor/ob_slice_calc.h +++ b/src/sql/executor/ob_slice_calc.h @@ -232,7 +232,7 @@ public: int init_partition_cache_map(); - virtual int init(); + virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID); virtual int destroy() { int ret = OB_SUCCESS; @@ -242,7 +242,7 @@ public: return ret; } - int build_repart_ch_map(ObPxPartChMap &map); + int build_repart_ch_map(ObPxPartChMap &map, uint64_t tenant_id); int build_part2tablet_id_map(); protected: // this is a trick! @@ -320,7 +320,7 @@ protected: {} ~ObSlaveMapRepartIdxCalcBase() = default; protected: - virtual int init() override; + virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override; virtual int destroy() override; protected: // 存储同一个partition所对应的所有task id @@ -390,7 +390,7 @@ public: virtual int get_slice_idx(const ObIArray &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx) override; - virtual int init() override; + virtual int init(uint64_t tenant_id) override; virtual int destroy() override; private: int get_task_idx_by_tablet_id(int64_t tablet_id , int64_t &task_idx); @@ -810,7 +810,7 @@ public: sort_cmp_(sort_cmp_funs, sort_collations) {} virtual ~ObSlaveMapPkeyRangeIdxCalc(); - virtual int init() override; + virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override; virtual int destroy() override; virtual int get_slice_idx( const common::ObIArray &exprs, @@ -932,7 +932,7 @@ public: } ~ObSlaveMapPkeyHashIdxCalc() = default; - int init() override; + int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override; int destroy() override; // for static engine @@ -1091,7 +1091,7 @@ public: } ~ObNullAwareAffinitizedRepartSliceIdxCalc() = default; - virtual int init() override; + virtual int init(uint64_t tenant_id) override; virtual int get_slice_idx(const ObIArray &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx) { UNUSED(exprs);