hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix se array of tablet range allocate too much memory and send piece msg core

Browse Source
This commit is contained in:
sdc
2023-07-24 07:18:34 +00:00
committed by ob-robot
parent 599e8f8035
commit 283a9c0b9a
10 changed files with 80 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deps/oblib/src/lib/container/ob_2d_array.h vendored
View File

@ -62,6 +62,7 @@ public:
inline const BlockAllocatorT &get_block_allocator() const { return block_alloc_; }
void set_tenant_id(int64_t tenant_id) { block_alloc_.set_tenant_id(tenant_id); }
void set_ctx_id(int64_t ctx_id) { block_alloc_.set_ctx_id(ctx_id); }
void set_label(const lib::ObLabel &label) { block_alloc_.set_label(label); };
virtual bool empty() const;

1
src/share/datum/ob_datum.h
View File

@ -371,6 +371,7 @@ public:
pack_ = static_cast<uint32_t>(length);//TODO(yuanzhi.zy):need check
}
inline void set_datum(const ObDatum &other) { *this = other; }
inline int64_t get_deep_copy_size() const { return is_null() ? 0 : len_; }
inline int deep_copy(const ObDatum &src, char *buf, int64_t max_size, int64_t &pos)
{
int ret = OB_SUCCESS;

20
src/sql/engine/basic/ob_stat_collector_op.cpp
View File

@ -17,6 +17,7 @@
#include "sql/engine/table/ob_block_sample_scan_op.h"
#include "sql/engine/table/ob_row_sample_scan_op.h"
#include "common/ob_smart_call.h"
#include "sql/engine/px/ob_px_sqc_handler.h"
using namespace oceanbase::common;
using namespace oceanbase::sql;
@ -273,6 +274,8 @@ int ObStatCollectorOp::split_partition_range()
int64_t border_vals_cnt = MY_SPEC.sort_exprs_.count() - (int64_t)(!is_none_partition());
ObPxTabletRange::DatumKey border_vals;
ObPxTabletRange partition_range;
Ob2DArray<ObPxTabletRange> tmp_part_ranges;
int64_t datum_len_sum = 0;
OZ(border_vals.prepare_allocate(border_vals_cnt));
while (OB_SUCC(ret) && !sort_iter_end) {
if (OB_FAIL(sort_impl_.get_next_row(MY_SPEC.sort_exprs_))) {
@ -291,7 +294,7 @@ int ObStatCollectorOp::split_partition_range()
LOG_WARN("fail to get partition id", K(ret));
} else {
if (pre_tablet_id != OB_INVALID_ID) {
OZ(ctx_.add_partition_range(partition_range));
OZ(tmp_part_ranges.push_back(partition_range));
partition_range.reset();
}
if (is_none_partition()) {
@ -303,7 +306,7 @@ int ObStatCollectorOp::split_partition_range()
}
}
if (OB_SUCC(ret) && sort_iter_end) {
OZ(ctx_.add_partition_range(partition_range));
OZ(tmp_part_ranges.push_back(partition_range));
partition_range.reset();
}
CK(0 != step);
@ -321,6 +324,8 @@ int ObStatCollectorOp::split_partition_range()
} else if (OB_FAIL(border_vals.at(i - (int64_t)!is_none_partition()).
deep_copy(*cur_datum, ctx_.get_allocator()))) {
LOG_WARN("deep copy datum failed", K(ret), K(i), K(*cur_datum));
} else {
datum_len_sum += cur_datum->get_deep_copy_size();
}
}
if (OB_SUCC(ret)) {
@ -331,6 +336,17 @@ int ObStatCollectorOp::split_partition_range()
}
}
}
if (OB_SUCC(ret) && OB_LIKELY(datum_len_sum > 0)) {
void *buf = NULL;
if (OB_ISNULL(ctx_.get_sqc_handler())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sqc handler is null", K(ret));
} else if (OB_ISNULL(buf = ctx_.get_sqc_handler()->get_safe_allocator().alloc(datum_len_sum))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), K(datum_len_sum));
} else {
}
}
}
return ret;
}

15
src/sql/engine/ob_exec_context.cpp
View File

@ -738,6 +738,8 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan)
const ObPhyPlanHint &phy_plan_hint = plan.get_phy_plan_hint();
ObConsistencyLevel consistency = INVALID_CONSISTENCY;
my_session_->set_cur_phy_plan(const_cast<ObPhysicalPlan*>(&plan));
part_ranges_.set_tenant_id(my_session_->get_effective_tenant_id());
part_ranges_.set_label("PxTabletRangArr");
if (OB_UNLIKELY(phy_plan_hint.query_timeout_ > 0)) {
plan_timeout = phy_plan_hint.query_timeout_;
} else {
@ -796,12 +798,8 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan)
return ret;
}
int ObExecContext::add_partition_range(ObPxTabletRange &part_range)
{
return part_ranges_.push_back(part_range);
}
int ObExecContext::set_partition_ranges(const ObIArray<ObPxTabletRange> &part_ranges)
int ObExecContext::set_partition_ranges(const Ob2DArray<ObPxTabletRange> &part_ranges,
char *buf, int64_t size)
{
int ret = OB_SUCCESS;
part_ranges_.reset();
@ -809,10 +807,13 @@ int ObExecContext::set_partition_ranges(const ObIArray<ObPxTabletRange> &part_ra
ret = OB_INVALID_ARGUMENT;
LOG_WARN("part ranges is empty", K(ret), K(part_ranges.count()));
} else {
int64_t pos = 0;
ObPxTabletRange tmp_range;
for (int64_t i = 0; OB_SUCC(ret) && i < part_ranges.count(); ++i) {
const ObPxTabletRange &cur_range = part_ranges.at(i);
if (OB_FAIL(tmp_range.deep_copy_from(cur_range, get_allocator()))) {
if (0 == size && OB_FAIL(tmp_range.deep_copy_from<true>(cur_range, get_allocator(), buf, size, pos))) {
LOG_WARN("deep copy partition range failed", K(ret), K(cur_range));
} else if (0 != size && OB_FAIL(tmp_range.deep_copy_from<false>(cur_range, get_allocator(), buf, size, pos))) {
LOG_WARN("deep copy partition range failed", K(ret), K(cur_range));
} else if (OB_FAIL(part_ranges_.push_back(tmp_range))) {
LOG_WARN("push back partition range failed", K(ret), K(tmp_range));

8
src/sql/engine/ob_exec_context.h
View File

@ -427,9 +427,9 @@ public:
PartitionIdCalcType get_partition_id_calc_type() { return calc_type_; }
void set_fixed_id(ObObjectID fixed_id) { fixed_id_ = fixed_id; }
ObObjectID get_fixed_id() { return fixed_id_; }
const ObIArray<ObPxTabletRange> &get_partition_ranges() const { return part_ranges_; }
int set_partition_ranges(const ObIArray<ObPxTabletRange> &part_ranges);
int add_partition_range(ObPxTabletRange &part_range);
const Ob2DArray<ObPxTabletRange> &get_partition_ranges() const { return part_ranges_; }
int set_partition_ranges(const Ob2DArray<ObPxTabletRange> &part_ranges,
char *buf = NULL, int64_t max_size = 0);
int fill_px_batch_info(
ObBatchRescanParams &params,
int64_t batch_id,
@ -602,7 +602,7 @@ protected:
ObObjectID fixed_id_; // fixed part id or fixed subpart ids
// sample result
ObSEArray<ObPxTabletRange, 1> part_ranges_;
Ob2DArray<ObPxTabletRange> part_ranges_;
int64_t check_status_times_;
ObIVirtualTableIteratorFactory *vt_ift_;

5
src/sql/engine/px/datahub/components/ob_dh_sample.cpp
View File

@ -183,10 +183,13 @@ int ObDynamicSampleWholeMsg::assign(const ObDynamicSampleWholeMsg &other, common
if (OB_FAIL(part_ranges_.reserve(other.part_ranges_.count()))) {
LOG_WARN("reserve partition ranges failed", K(ret), K(other.part_ranges_.count()));
}
char *buf = NULL;
int64_t size = 0;
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < other.part_ranges_.count(); ++i) {
const ObPxTabletRange &cur_part_range = other.part_ranges_.at(i);
ObPxTabletRange tmp_part_range;
if (OB_FAIL(tmp_part_range.deep_copy_from(cur_part_range, *allocator))) {
if (OB_FAIL(tmp_part_range.deep_copy_from<true>(cur_part_range, *allocator, buf, size, pos))) {
LOG_WARN("deep copy partition range failed", K(ret), K(cur_part_range), K(i));
} else if (OB_FAIL(part_ranges_.push_back(tmp_part_range))) {
LOG_WARN("push back partition range failed", K(ret), K(tmp_part_range), K(i));

4
src/sql/engine/px/datahub/components/ob_dh_sample.h
View File

@ -61,7 +61,7 @@ public:
int64_t expect_range_count_;
ObSEArray<uint64_t, 1> tablet_ids_;
ObPxSampleType sample_type_;
ObSEArray<ObPxTabletRange, 1> part_ranges_;
Ob2DArray<ObPxTabletRange> part_ranges_;
ObArray<ObChunkDatumStore *> row_stores_;
ObArenaAllocator arena_; // for deserialize
common::ObSpinLock spin_lock_; // for merge piece msg
@ -83,7 +83,7 @@ public:
INHERIT_TO_STRING_KV("meta", ObDatahubWholeMsg<dtl::ObDtlMsgType::DH_DYNAMIC_SAMPLE_WHOLE_MSG>,
K_(op_id), K_(part_ranges));
public:
common::ObSEArray<ObPxTabletRange, OB_DEFAULT_SE_ARRAY_COUNT> part_ranges_;
common::Ob2DArray<ObPxTabletRange> part_ranges_;
common::ObArenaAllocator assign_allocator_;
private:
DISALLOW_COPY_AND_ASSIGN(ObDynamicSampleWholeMsg);

31
src/sql/engine/px/ob_px_dtl_msg.cpp
View File

@ -89,37 +89,6 @@ bool ObPxTabletRange::is_valid() const
return tablet_id_ >= 0 && range_cut_.count() >= 0;
}
int ObPxTabletRange::deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
reset();
tablet_id_ = other.tablet_id_;
range_weights_ = other.range_weights_;
if (OB_FAIL(range_cut_.reserve(other.range_cut_.count()))) {
LOG_WARN("reserve end keys failed", K(ret), K(other.range_cut_.count()));
}
DatumKey copied_key;
RangeWeight range_weight;
ObDatum tmp_datum;
for (int64_t i = 0; OB_SUCC(ret) && i < other.range_cut_.count(); ++i) {
const DatumKey &cur_key = other.range_cut_.at(i);
copied_key.reuse();
range_weight.reuse();
for (int64_t j = 0; OB_SUCC(ret) && j < cur_key.count(); ++j) {
if (OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), allocator))) {
LOG_WARN("deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)));
} else if (OB_FAIL(copied_key.push_back(tmp_datum))) {
LOG_WARN("push back datum failed", K(ret), K(i), K(j), K(tmp_datum));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(range_cut_.push_back(copied_key))) {
LOG_WARN("push back rowkey failed", K(ret), K(copied_key), K(i));
}
}
return ret;
}
int ObPxTabletRange::assign(const ObPxTabletRange &other)
{
int ret = OB_SUCCESS;

41
src/sql/engine/px/ob_px_dtl_msg.h
View File

@ -527,7 +527,9 @@ public:
~ObPxTabletRange() = default;
void reset();
bool is_valid() const;
int deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator);
template <bool use_allocator>
int deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator,
char *buf, int64_t size, int64_t &pos);
int assign(const ObPxTabletRange &other);
int64_t get_range_col_cnt() const { return range_cut_.empty() ? 0 :
range_cut_.at(0).count(); }
@ -535,7 +537,7 @@ public:
public:
static const int64_t DEFAULT_RANGE_COUNT = 8;
typedef common::ObSEArray<common::ObRowkey, DEFAULT_RANGE_COUNT> EndKeys;
typedef ObSEArray<ObDatum, 2> DatumKey;
typedef Ob2DArray<ObDatum> DatumKey;
typedef ObSEArray<int64_t, 2> RangeWeight;
typedef ObSEArray<DatumKey, DEFAULT_RANGE_COUNT> RangeCut; // not include MAX at last nor MIN at first
typedef ObSEArray<RangeWeight, DEFAULT_RANGE_COUNT> RangeWeights;
@ -545,6 +547,41 @@ public:
RangeCut range_cut_;
};
template <bool use_allocator>
int ObPxTabletRange::deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator,
char *buf, int64_t size, int64_t &pos)
{
int ret = OB_SUCCESS;
reset();
tablet_id_ = other.tablet_id_;
range_weights_ = other.range_weights_;
if (OB_FAIL(range_cut_.reserve(other.range_cut_.count()))) {
SQL_LOG(WARN, "reserve end keys failed", K(ret), K(other.range_cut_.count()));
}
DatumKey copied_key;
RangeWeight range_weight;
ObDatum tmp_datum;
for (int64_t i = 0; OB_SUCC(ret) && i < other.range_cut_.count(); ++i) {
const DatumKey &cur_key = other.range_cut_.at(i);
copied_key.reuse();
range_weight.reuse();
for (int64_t j = 0; OB_SUCC(ret) && j < cur_key.count(); ++j) {
if (use_allocator && OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), allocator))) {
SQL_LOG(WARN, "deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)));
} else if (!use_allocator && OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), buf, size, pos))) {
SQL_LOG(WARN, "deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)), K(size), K(pos));
} else if (OB_FAIL(copied_key.push_back(tmp_datum))) {
SQL_LOG(WARN, "push back datum failed", K(ret), K(i), K(j), K(tmp_datum));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(range_cut_.push_back(copied_key))) {
SQL_LOG(WARN, "push back rowkey failed", K(ret), K(copied_key), K(i));
}
}
return ret;
}
}
}
#endif /*_OB_SQL_PX_DTL_MSG_H_ */

6
src/sql/executor/ob_slice_calc.cpp
View File

@ -859,7 +859,7 @@ int ObSlaveMapPkeyRangeIdxCalc::build_partition_range_channel_map(
{
int ret = OB_SUCCESS;
part_range_channel_map.destroy();
const ObIArray<ObPxTabletRange> &part_ranges = exec_ctx_.get_partition_ranges();
const Ob2DArray<ObPxTabletRange> &part_ranges = exec_ctx_.get_partition_ranges();
if (OB_FAIL(part_range_channel_map.create(DEFAULT_PARTITION_COUNT, common::ObModIds::OB_SQL_PX))) {
LOG_WARN("create part range map failed", K(ret));
} else {
@ -951,7 +951,7 @@ bool ObSlaveMapPkeyRangeIdxCalc::Compare::operator()(
int cmp = 0;
const int64_t cnt = sort_cmp_funs_->count();
for (int64_t i = 0; OB_SUCC(ret) && 0 == cmp && i < cnt; i++) {
if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l[i], r[i], cmp))) {
if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l.at(i), r.at(i), cmp))) {
LOG_WARN("do cmp failed", K(ret), K(i), K(l), K(r));
} else if (cmp < 0) {
less = sort_collations_->at(i).is_ascending_;
@ -1266,7 +1266,7 @@ bool ObRangeSliceIdCalc::Compare::operator()(
int cmp = 0;
const int64_t cnt = sort_cmp_funs_->count();
for (int64_t i = 0; OB_SUCC(ret) && 0 == cmp && i < cnt; i++) {
if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l[i], r[i], cmp))) {
if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l.at(i), r.at(i), cmp))) {
LOG_WARN("do cmp failed", K(ret), K(i), K(l), K(r));
} else if (cmp < 0) {
less = sort_collations_->at(i).is_ascending_;