hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix se array of tablet range allocate too much memory and send piece msg core

Browse Source
This commit is contained in:
sdc
2023-07-24 07:18:34 +00:00
committed by ob-robot
parent 599e8f8035
commit 283a9c0b9a
10 changed files with 80 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/sql/engine/px/datahub/components/ob_dh_sample.cpp
View File

@ -183,10 +183,13 @@ int ObDynamicSampleWholeMsg::assign(const ObDynamicSampleWholeMsg &other, common
if (OB_FAIL(part_ranges_.reserve(other.part_ranges_.count()))) {
LOG_WARN("reserve partition ranges failed", K(ret), K(other.part_ranges_.count()));
}
char *buf = NULL;
int64_t size = 0;
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < other.part_ranges_.count(); ++i) {
const ObPxTabletRange &cur_part_range = other.part_ranges_.at(i);
ObPxTabletRange tmp_part_range;
if (OB_FAIL(tmp_part_range.deep_copy_from(cur_part_range, *allocator))) {
if (OB_FAIL(tmp_part_range.deep_copy_from<true>(cur_part_range, *allocator, buf, size, pos))) {
LOG_WARN("deep copy partition range failed", K(ret), K(cur_part_range), K(i));
} else if (OB_FAIL(part_ranges_.push_back(tmp_part_range))) {
LOG_WARN("push back partition range failed", K(ret), K(tmp_part_range), K(i));

4
src/sql/engine/px/datahub/components/ob_dh_sample.h
View File

@ -61,7 +61,7 @@ public:
int64_t expect_range_count_;
ObSEArray<uint64_t, 1> tablet_ids_;
ObPxSampleType sample_type_;
ObSEArray<ObPxTabletRange, 1> part_ranges_;
Ob2DArray<ObPxTabletRange> part_ranges_;
ObArray<ObChunkDatumStore *> row_stores_;
ObArenaAllocator arena_; // for deserialize
common::ObSpinLock spin_lock_; // for merge piece msg
@ -83,7 +83,7 @@ public:
INHERIT_TO_STRING_KV("meta", ObDatahubWholeMsg<dtl::ObDtlMsgType::DH_DYNAMIC_SAMPLE_WHOLE_MSG>,
K_(op_id), K_(part_ranges));
public:
common::ObSEArray<ObPxTabletRange, OB_DEFAULT_SE_ARRAY_COUNT> part_ranges_;
common::Ob2DArray<ObPxTabletRange> part_ranges_;
common::ObArenaAllocator assign_allocator_;
private:
DISALLOW_COPY_AND_ASSIGN(ObDynamicSampleWholeMsg);

31
src/sql/engine/px/ob_px_dtl_msg.cpp
View File

@ -89,37 +89,6 @@ bool ObPxTabletRange::is_valid() const
return tablet_id_ >= 0 && range_cut_.count() >= 0;
}
int ObPxTabletRange::deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
reset();
tablet_id_ = other.tablet_id_;
range_weights_ = other.range_weights_;
if (OB_FAIL(range_cut_.reserve(other.range_cut_.count()))) {
LOG_WARN("reserve end keys failed", K(ret), K(other.range_cut_.count()));
}
DatumKey copied_key;
RangeWeight range_weight;
ObDatum tmp_datum;
for (int64_t i = 0; OB_SUCC(ret) && i < other.range_cut_.count(); ++i) {
const DatumKey &cur_key = other.range_cut_.at(i);
copied_key.reuse();
range_weight.reuse();
for (int64_t j = 0; OB_SUCC(ret) && j < cur_key.count(); ++j) {
if (OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), allocator))) {
LOG_WARN("deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)));
} else if (OB_FAIL(copied_key.push_back(tmp_datum))) {
LOG_WARN("push back datum failed", K(ret), K(i), K(j), K(tmp_datum));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(range_cut_.push_back(copied_key))) {
LOG_WARN("push back rowkey failed", K(ret), K(copied_key), K(i));
}
}
return ret;
}
int ObPxTabletRange::assign(const ObPxTabletRange &other)
{
int ret = OB_SUCCESS;

41
src/sql/engine/px/ob_px_dtl_msg.h
View File

@ -527,7 +527,9 @@ public:
~ObPxTabletRange() = default;
void reset();
bool is_valid() const;
int deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator);
template <bool use_allocator>
int deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator,
char *buf, int64_t size, int64_t &pos);
int assign(const ObPxTabletRange &other);
int64_t get_range_col_cnt() const { return range_cut_.empty() ? 0 :
range_cut_.at(0).count(); }
@ -535,7 +537,7 @@ public:
public:
static const int64_t DEFAULT_RANGE_COUNT = 8;
typedef common::ObSEArray<common::ObRowkey, DEFAULT_RANGE_COUNT> EndKeys;
typedef ObSEArray<ObDatum, 2> DatumKey;
typedef Ob2DArray<ObDatum> DatumKey;
typedef ObSEArray<int64_t, 2> RangeWeight;
typedef ObSEArray<DatumKey, DEFAULT_RANGE_COUNT> RangeCut; // not include MAX at last nor MIN at first
typedef ObSEArray<RangeWeight, DEFAULT_RANGE_COUNT> RangeWeights;
@ -545,6 +547,41 @@ public:
RangeCut range_cut_;
};
template <bool use_allocator>
int ObPxTabletRange::deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator,
char *buf, int64_t size, int64_t &pos)
{
int ret = OB_SUCCESS;
reset();
tablet_id_ = other.tablet_id_;
range_weights_ = other.range_weights_;
if (OB_FAIL(range_cut_.reserve(other.range_cut_.count()))) {
SQL_LOG(WARN, "reserve end keys failed", K(ret), K(other.range_cut_.count()));
}
DatumKey copied_key;
RangeWeight range_weight;
ObDatum tmp_datum;
for (int64_t i = 0; OB_SUCC(ret) && i < other.range_cut_.count(); ++i) {
const DatumKey &cur_key = other.range_cut_.at(i);
copied_key.reuse();
range_weight.reuse();
for (int64_t j = 0; OB_SUCC(ret) && j < cur_key.count(); ++j) {
if (use_allocator && OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), allocator))) {
SQL_LOG(WARN, "deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)));
} else if (!use_allocator && OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), buf, size, pos))) {
SQL_LOG(WARN, "deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)), K(size), K(pos));
} else if (OB_FAIL(copied_key.push_back(tmp_datum))) {
SQL_LOG(WARN, "push back datum failed", K(ret), K(i), K(j), K(tmp_datum));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(range_cut_.push_back(copied_key))) {
SQL_LOG(WARN, "push back rowkey failed", K(ret), K(copied_key), K(i));
}
}
return ret;
}
}
}
#endif /*_OB_SQL_PX_DTL_MSG_H_ */