fix se array of tablet range allocate too much memory and send piece msg core

deps/oblib/src/lib/container/ob_2d_array.h

@@ -62,6 +62,7 @@ public:
   inline const BlockAllocatorT &get_block_allocator() const { return block_alloc_; }
   void set_tenant_id(int64_t tenant_id) { block_alloc_.set_tenant_id(tenant_id); }
   void set_ctx_id(int64_t ctx_id) { block_alloc_.set_ctx_id(ctx_id); }
+  void set_label(const lib::ObLabel &label) { block_alloc_.set_label(label); };
 
   virtual bool empty() const;
 

src/share/datum/ob_datum.h

@@ -371,6 +371,7 @@ public:
     pack_ = static_cast<uint32_t>(length);//TODO(yuanzhi.zy):need check
   }
   inline void set_datum(const ObDatum &other) { *this = other; }
+  inline int64_t get_deep_copy_size() const { return is_null() ? 0 : len_; }
   inline int deep_copy(const ObDatum &src, char *buf, int64_t max_size, int64_t &pos)
   {
     int ret = OB_SUCCESS;

src/sql/engine/basic/ob_stat_collector_op.cpp

@@ -17,6 +17,7 @@
 #include "sql/engine/table/ob_block_sample_scan_op.h"
 #include "sql/engine/table/ob_row_sample_scan_op.h"
 #include "common/ob_smart_call.h"
+#include "sql/engine/px/ob_px_sqc_handler.h"
 
 using namespace oceanbase::common;
 using namespace oceanbase::sql;
@@ -273,6 +274,8 @@ int ObStatCollectorOp::split_partition_range()
     int64_t border_vals_cnt = MY_SPEC.sort_exprs_.count() - (int64_t)(!is_none_partition());
     ObPxTabletRange::DatumKey border_vals;
     ObPxTabletRange partition_range;
+    Ob2DArray<ObPxTabletRange> tmp_part_ranges;
+    int64_t datum_len_sum = 0;
     OZ(border_vals.prepare_allocate(border_vals_cnt));
     while (OB_SUCC(ret) && !sort_iter_end) {
       if (OB_FAIL(sort_impl_.get_next_row(MY_SPEC.sort_exprs_))) {
@@ -291,7 +294,7 @@ int ObStatCollectorOp::split_partition_range()
           LOG_WARN("fail to get partition id", K(ret));
         } else {
           if (pre_tablet_id != OB_INVALID_ID) {
-            OZ(ctx_.add_partition_range(partition_range));
+            OZ(tmp_part_ranges.push_back(partition_range));
             partition_range.reset();
           }
           if (is_none_partition()) {
@@ -303,7 +306,7 @@ int ObStatCollectorOp::split_partition_range()
         }
       }
       if (OB_SUCC(ret) && sort_iter_end) {
-        OZ(ctx_.add_partition_range(partition_range));
+        OZ(tmp_part_ranges.push_back(partition_range));
         partition_range.reset();
       }
       CK(0 != step);
@@ -321,6 +324,8 @@ int ObStatCollectorOp::split_partition_range()
             } else if (OB_FAIL(border_vals.at(i - (int64_t)!is_none_partition()).
                   deep_copy(*cur_datum, ctx_.get_allocator()))) {
               LOG_WARN("deep copy datum failed", K(ret), K(i), K(*cur_datum));
+            } else {
+              datum_len_sum += cur_datum->get_deep_copy_size();
             }
           }
           if (OB_SUCC(ret)) {
@@ -331,6 +336,17 @@ int ObStatCollectorOp::split_partition_range()
         }
       }
     }
+    if (OB_SUCC(ret) && OB_LIKELY(datum_len_sum > 0)) {
+      void *buf = NULL;
+      if (OB_ISNULL(ctx_.get_sqc_handler())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("sqc handler is null", K(ret));
+      } else if (OB_ISNULL(buf = ctx_.get_sqc_handler()->get_safe_allocator().alloc(datum_len_sum))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        LOG_WARN("allocate memory failed", K(ret), K(datum_len_sum));
+      } else {
+      }
+    }
   }
   return ret;
 }

src/sql/engine/ob_exec_context.cpp

@@ -738,6 +738,8 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan)
     const ObPhyPlanHint &phy_plan_hint = plan.get_phy_plan_hint();
     ObConsistencyLevel consistency = INVALID_CONSISTENCY;
     my_session_->set_cur_phy_plan(const_cast<ObPhysicalPlan*>(&plan));
+    part_ranges_.set_tenant_id(my_session_->get_effective_tenant_id());
+    part_ranges_.set_label("PxTabletRangArr");
     if (OB_UNLIKELY(phy_plan_hint.query_timeout_ > 0)) {
       plan_timeout = phy_plan_hint.query_timeout_;
     } else {
@@ -796,12 +798,8 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan)
   return ret;
 }
 
-int ObExecContext::add_partition_range(ObPxTabletRange &part_range)
+int ObExecContext::set_partition_ranges(const Ob2DArray<ObPxTabletRange> &part_ranges,
-{
+                                        char *buf, int64_t size)
-  return part_ranges_.push_back(part_range);
-}
-
-int ObExecContext::set_partition_ranges(const ObIArray<ObPxTabletRange> &part_ranges)
 {
   int ret = OB_SUCCESS;
   part_ranges_.reset();
@@ -809,10 +807,13 @@ int ObExecContext::set_partition_ranges(const ObIArray<ObPxTabletRange> &part_ra
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("part ranges is empty", K(ret), K(part_ranges.count()));
   } else {
+    int64_t pos = 0;
     ObPxTabletRange tmp_range;
     for (int64_t i = 0; OB_SUCC(ret) && i < part_ranges.count(); ++i) {
       const ObPxTabletRange &cur_range = part_ranges.at(i);
-      if (OB_FAIL(tmp_range.deep_copy_from(cur_range, get_allocator()))) {
+      if (0 == size && OB_FAIL(tmp_range.deep_copy_from<true>(cur_range, get_allocator(), buf, size, pos))) {
+        LOG_WARN("deep copy partition range failed", K(ret), K(cur_range));
+      } else if (0 != size && OB_FAIL(tmp_range.deep_copy_from<false>(cur_range, get_allocator(), buf, size, pos))) {
         LOG_WARN("deep copy partition range failed", K(ret), K(cur_range));
       } else if (OB_FAIL(part_ranges_.push_back(tmp_range))) {
         LOG_WARN("push back partition range failed", K(ret), K(tmp_range));

src/sql/engine/ob_exec_context.h

@@ -427,9 +427,9 @@ public:
   PartitionIdCalcType get_partition_id_calc_type() { return calc_type_; }
   void set_fixed_id(ObObjectID fixed_id) { fixed_id_ = fixed_id; }
   ObObjectID get_fixed_id() { return fixed_id_; }
-  const ObIArray<ObPxTabletRange> &get_partition_ranges() const { return part_ranges_; }
+  const Ob2DArray<ObPxTabletRange> &get_partition_ranges() const { return part_ranges_; }
-  int set_partition_ranges(const ObIArray<ObPxTabletRange> &part_ranges);
+  int set_partition_ranges(const Ob2DArray<ObPxTabletRange> &part_ranges,
-  int add_partition_range(ObPxTabletRange &part_range);
+                           char *buf = NULL, int64_t max_size = 0);
   int fill_px_batch_info(
       ObBatchRescanParams &params,
       int64_t batch_id,
@@ -602,7 +602,7 @@ protected:
   ObObjectID fixed_id_;    // fixed part id or fixed subpart ids
 
   // sample result
-  ObSEArray<ObPxTabletRange, 1> part_ranges_;
+  Ob2DArray<ObPxTabletRange> part_ranges_;
   int64_t check_status_times_;
   ObIVirtualTableIteratorFactory *vt_ift_;
 

src/sql/engine/px/datahub/components/ob_dh_sample.cpp

@@ -183,10 +183,13 @@ int ObDynamicSampleWholeMsg::assign(const ObDynamicSampleWholeMsg &other, common
   if (OB_FAIL(part_ranges_.reserve(other.part_ranges_.count()))) {
     LOG_WARN("reserve partition ranges failed", K(ret), K(other.part_ranges_.count()));
   }
+  char *buf = NULL;
+  int64_t size = 0;
+  int64_t pos = 0;
   for (int64_t i = 0; OB_SUCC(ret) && i < other.part_ranges_.count(); ++i) {
     const ObPxTabletRange &cur_part_range = other.part_ranges_.at(i);
     ObPxTabletRange tmp_part_range;
-    if (OB_FAIL(tmp_part_range.deep_copy_from(cur_part_range, *allocator))) {
+    if (OB_FAIL(tmp_part_range.deep_copy_from<true>(cur_part_range, *allocator, buf, size, pos))) {
       LOG_WARN("deep copy partition range failed", K(ret), K(cur_part_range), K(i));
     } else if (OB_FAIL(part_ranges_.push_back(tmp_part_range))) {
       LOG_WARN("push back partition range failed", K(ret), K(tmp_part_range), K(i));

src/sql/engine/px/datahub/components/ob_dh_sample.h

@@ -61,7 +61,7 @@ public:
   int64_t expect_range_count_;
   ObSEArray<uint64_t, 1> tablet_ids_;
   ObPxSampleType sample_type_;
-  ObSEArray<ObPxTabletRange, 1> part_ranges_;
+  Ob2DArray<ObPxTabletRange> part_ranges_;
   ObArray<ObChunkDatumStore *> row_stores_;
   ObArenaAllocator arena_; // for deserialize
   common::ObSpinLock spin_lock_; // for merge piece msg
@@ -83,7 +83,7 @@ public:
   INHERIT_TO_STRING_KV("meta", ObDatahubWholeMsg<dtl::ObDtlMsgType::DH_DYNAMIC_SAMPLE_WHOLE_MSG>,
                        K_(op_id), K_(part_ranges));
 public:
-  common::ObSEArray<ObPxTabletRange, OB_DEFAULT_SE_ARRAY_COUNT> part_ranges_;
+  common::Ob2DArray<ObPxTabletRange> part_ranges_;
   common::ObArenaAllocator assign_allocator_;
 private:
   DISALLOW_COPY_AND_ASSIGN(ObDynamicSampleWholeMsg);

src/sql/engine/px/ob_px_dtl_msg.cpp

@@ -89,37 +89,6 @@ bool ObPxTabletRange::is_valid() const
   return tablet_id_ >= 0 && range_cut_.count() >= 0;
 }
 
-int ObPxTabletRange::deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator)
-{
-  int ret = OB_SUCCESS;
-  reset();
-  tablet_id_ = other.tablet_id_;
-  range_weights_ = other.range_weights_;
-  if (OB_FAIL(range_cut_.reserve(other.range_cut_.count()))) {
-    LOG_WARN("reserve end keys failed", K(ret), K(other.range_cut_.count()));
-  }
-  DatumKey copied_key;
-  RangeWeight range_weight;
-  ObDatum tmp_datum;
-  for (int64_t i = 0; OB_SUCC(ret) && i < other.range_cut_.count(); ++i) {
-    const DatumKey &cur_key = other.range_cut_.at(i);
-    copied_key.reuse();
-    range_weight.reuse();
-    for (int64_t j = 0; OB_SUCC(ret) && j < cur_key.count(); ++j) {
-      if (OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), allocator))) {
-        LOG_WARN("deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)));
-      } else if (OB_FAIL(copied_key.push_back(tmp_datum))) {
-        LOG_WARN("push back datum failed", K(ret), K(i), K(j), K(tmp_datum));
-      }
-    }
-    if (OB_FAIL(ret)) {
-    } else if (OB_FAIL(range_cut_.push_back(copied_key))) {
-      LOG_WARN("push back rowkey failed", K(ret), K(copied_key), K(i));
-    }
-  }
-  return ret;
-}
-
 int ObPxTabletRange::assign(const ObPxTabletRange &other)
 {
   int ret = OB_SUCCESS;

src/sql/engine/px/ob_px_dtl_msg.h

@@ -527,7 +527,9 @@ public:
   ~ObPxTabletRange() = default;
   void reset();
   bool is_valid() const;
-  int deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator);
+  template <bool use_allocator>
+  int deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator,
+                     char *buf, int64_t size, int64_t &pos);
   int assign(const ObPxTabletRange &other);
   int64_t get_range_col_cnt() const { return range_cut_.empty() ? 0 :
       range_cut_.at(0).count(); }
@@ -535,7 +537,7 @@ public:
 public:
   static const int64_t DEFAULT_RANGE_COUNT = 8;
   typedef common::ObSEArray<common::ObRowkey, DEFAULT_RANGE_COUNT> EndKeys;
-  typedef ObSEArray<ObDatum, 2> DatumKey;
+  typedef Ob2DArray<ObDatum> DatumKey;
   typedef ObSEArray<int64_t, 2> RangeWeight;
   typedef ObSEArray<DatumKey, DEFAULT_RANGE_COUNT> RangeCut; // not include MAX at last nor MIN at first
   typedef ObSEArray<RangeWeight, DEFAULT_RANGE_COUNT> RangeWeights;
@@ -545,6 +547,41 @@ public:
   RangeCut range_cut_;
 };
 
+template <bool use_allocator>
+int ObPxTabletRange::deep_copy_from(const ObPxTabletRange &other, common::ObIAllocator &allocator,
+                                    char *buf, int64_t size, int64_t &pos)
+{
+  int ret = OB_SUCCESS;
+  reset();
+  tablet_id_ = other.tablet_id_;
+  range_weights_ = other.range_weights_;
+  if (OB_FAIL(range_cut_.reserve(other.range_cut_.count()))) {
+    SQL_LOG(WARN, "reserve end keys failed", K(ret), K(other.range_cut_.count()));
+  }
+  DatumKey copied_key;
+  RangeWeight range_weight;
+  ObDatum tmp_datum;
+  for (int64_t i = 0; OB_SUCC(ret) && i < other.range_cut_.count(); ++i) {
+    const DatumKey &cur_key = other.range_cut_.at(i);
+    copied_key.reuse();
+    range_weight.reuse();
+    for (int64_t j = 0; OB_SUCC(ret) && j < cur_key.count(); ++j) {
+      if (use_allocator && OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), allocator))) {
+        SQL_LOG(WARN, "deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)));
+      } else if (!use_allocator && OB_FAIL(tmp_datum.deep_copy(cur_key.at(j), buf, size, pos))) {
+        SQL_LOG(WARN, "deep copy datum failed", K(ret), K(i), K(j), K(cur_key.at(j)), K(size), K(pos));
+      } else if (OB_FAIL(copied_key.push_back(tmp_datum))) {
+        SQL_LOG(WARN, "push back datum failed", K(ret), K(i), K(j), K(tmp_datum));
+      }
+    }
+    if (OB_FAIL(ret)) {
+    } else if (OB_FAIL(range_cut_.push_back(copied_key))) {
+      SQL_LOG(WARN, "push back rowkey failed", K(ret), K(copied_key), K(i));
+    }
+  }
+  return ret;
+}
+
 }
 }
 #endif /*_OB_SQL_PX_DTL_MSG_H_ */

src/sql/executor/ob_slice_calc.cpp

@@ -859,7 +859,7 @@ int ObSlaveMapPkeyRangeIdxCalc::build_partition_range_channel_map(
 {
   int ret = OB_SUCCESS;
   part_range_channel_map.destroy();
-  const ObIArray<ObPxTabletRange> &part_ranges = exec_ctx_.get_partition_ranges();
+  const Ob2DArray<ObPxTabletRange> &part_ranges = exec_ctx_.get_partition_ranges();
   if (OB_FAIL(part_range_channel_map.create(DEFAULT_PARTITION_COUNT, common::ObModIds::OB_SQL_PX))) {
     LOG_WARN("create part range map failed", K(ret));
   } else {
@@ -951,7 +951,7 @@ bool ObSlaveMapPkeyRangeIdxCalc::Compare::operator()(
     int cmp = 0;
     const int64_t cnt = sort_cmp_funs_->count();
     for (int64_t i = 0; OB_SUCC(ret) && 0 == cmp && i < cnt; i++) {
-      if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l[i], r[i], cmp))) {
+      if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l.at(i), r.at(i), cmp))) {
         LOG_WARN("do cmp failed", K(ret), K(i), K(l), K(r));
       } else if (cmp < 0) {
         less = sort_collations_->at(i).is_ascending_;
@@ -1266,7 +1266,7 @@ bool ObRangeSliceIdCalc::Compare::operator()(
     int cmp = 0;
     const int64_t cnt = sort_cmp_funs_->count();
     for (int64_t i = 0; OB_SUCC(ret) && 0 == cmp && i < cnt; i++) {
-      if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l[i], r[i], cmp))) {
+      if (OB_FAIL(sort_cmp_funs_->at(i).cmp_func_(l.at(i), r.at(i), cmp))) {
         LOG_WARN("do cmp failed", K(ret), K(i), K(l), K(r));
       } else if (cmp < 0) {
         less = sort_collations_->at(i).is_ascending_;