[BUGFIX] fix lob inrow data

deps/oblib/src/common/object/ob_obj_compare.cpp

@@ -1417,7 +1417,7 @@ int ObObjCmpFuncs::cmp_func<ObEnumSetTC, ObUIntTC>(const ObObj &obj1, \
       ObString wkb2 = obj2.get_string();                                                        \
       ObLobLocatorV2 lob1(wkb1, obj1.has_lob_header());                                         \
       ObLobLocatorV2 lob2(wkb2, obj2.has_lob_header());                                         \
-      if (lob1.is_inrow() && lob2.is_inrow()) {                                                 \
+      if (lob1.has_inrow_data() && lob2.has_inrow_data()) {                                     \
         (void)lob1.get_inrow_data(wkb1);                                                        \
         (void)lob2.get_inrow_data(wkb2);                                                        \
         cmp_ret = static_cast<int>(ObCharset::strcmpsp(CS_TYPE_BINARY,                          \
@@ -1450,7 +1450,7 @@ int ObObjCmpFuncs::cmp_func<ObEnumSetTC, ObUIntTC>(const ObObj &obj1, \
     ObString wkb2 = obj2.get_string();                                                          \
     ObLobLocatorV2 lob1(wkb1, obj1.has_lob_header());                                           \
     ObLobLocatorV2 lob2(wkb2, obj2.has_lob_header());                                           \
-    if (lob1.is_inrow() && lob2.is_inrow()) {                                                   \
+    if (lob1.has_inrow_data() && lob2.has_inrow_data()) {                                       \
       (void)lob1.get_inrow_data(wkb1);                                                          \
       (void)lob2.get_inrow_data(wkb2);                                                          \
     }                                                                                           \

deps/oblib/src/common/object/ob_obj_funcs.h

@@ -1541,7 +1541,7 @@ DEF_TEXT_FUNCS(ObLongTextType, string, ObString);
       if (!lob.is_valid()) {                                                         \
         COMMON_LOG(WARN, "invalid lob", K(ret), K(str));                             \
         right_to_die_or_duty_to_live();                                              \
-      } else if (!lob.is_inrow()) {                                                  \
+      } else if (!lob.has_inrow_data()) {                                            \
         COMMON_LOG(WARN, "meet outrow lob do calc hash value", K(lob));              \
         hash_res = hash;                                                             \
       } else if (OB_FAIL(lob.get_inrow_data(wkb))) {                                 \
@@ -1616,7 +1616,7 @@ DEF_GEO_FUNCS(ObGeometryType, string, ObString);
       if (!lob.is_valid()) {                                                         \
         COMMON_LOG(WARN, "invalid lob", K(ret), K(str));                             \
         right_to_die_or_duty_to_live();                                              \
-      } else if (!lob.is_inrow()) {                                                  \
+      } else if (!lob.has_inrow_data()) {                                            \
         COMMON_LOG(WARN, "meet outrow lob do calc hash value", K(lob));              \
         hash_res = hash;                                                             \
       } else if (OB_FAIL(lob.get_inrow_data(j_bin_str))) {                           \

deps/oblib/src/common/object/ob_object.cpp

@@ -449,9 +449,15 @@ int ObLobLocatorV2::fill(ObMemLobType type,
             K(ret), K(type), K(disk_lob_full_size), K(sizeof(ObLobCommon)));
         } else {
           uint32_t disk_loc_header_size = sizeof(ObLobCommon);
+          if (disk_loc->in_row_) {
             if (disk_loc->is_init_) {
               disk_loc_header_size += sizeof(ObLobData);
             }
+          } else {
+            int64_t tbz = disk_loc->get_byte_size(disk_lob_full_size);
+            int64_t thz = disk_loc->get_handle_size(tbz);
+            disk_loc_header_size = thz;
+          }
           if (offset + disk_loc_header_size > size_ || disk_lob_full_size < disk_loc_header_size) {
             ret = OB_INVALID_ARGUMENT;
             COMMON_LOG(WARN, "Lob: invalid disk locator",
@@ -564,7 +570,11 @@ int ObLobLocatorV2::get_disk_locator(ObString &disc_loc_buff) const
       ret = OB_ERR_UNEXPECTED;
       LOG_WARN("get invalid handle size", K(ret), K(size_), K(disk_loc), K(ptr_));
     } else {
+      if (disk_loc->in_row_) {
         handle_size = size_ - handle_size;
+      } else {
+        handle_size = disk_loc->get_handle_size(0);
+      }
       disc_loc_buff.assign_ptr(reinterpret_cast<const char *>(disk_loc), handle_size);
     }
   }
@@ -744,10 +754,14 @@ int ObLobLocatorV2::get_real_locator_len(int64_t &real_len) const
     COMMON_LOG(WARN, "Lob: get disk locator failed", K(ret), K(*this));
   } else {
     real_len = (uintptr_t)disk_loc - (uintptr_t)ptr_;
+    if (disk_loc->in_row_) {
       real_len += sizeof(ObLobCommon);
       if (disk_loc->is_init_) {
         real_len += sizeof(ObLobData);
       }
+    } else {
+      real_len += disk_loc->get_handle_size(0);
+    }
   }
   return ret;
 }
@@ -772,8 +786,9 @@ int ObLobLocatorV2::set_payload_data(const ObString& payload)
     } else {
       ObString disk_loc_buff;
       if (OB_SUCC(get_disk_locator(disk_loc_buff))) {
-        OB_ASSERT(payload.length() == disk_loc_buff.length());
+        buf_len = size_ - (disk_loc_buff.ptr() - ptr_);
-        MEMCPY(disk_loc_buff.ptr(), payload.ptr(), disk_loc_buff.length());
+        OB_ASSERT(payload.length() == buf_len);
+        MEMCPY(disk_loc_buff.ptr(), payload.ptr(), payload.length());
       }
     }
   }
@@ -804,7 +819,7 @@ int ObLobLocatorV2::set_payload_data(const ObLobCommon *lob_comm, const ObString
     if (loc->has_extern()) {
       if (OB_SUCC(get_disk_locator(disk_loc_buff))) {
         buf = disk_loc_buff.ptr();
-        buf_len = disk_loc_buff.length();
+        buf_len = (size_ - (disk_loc_buff.ptr() - ptr_));
       }
     } else if (!loc->has_extern()) {
       buf = loc->data_;
@@ -812,7 +827,11 @@ int ObLobLocatorV2::set_payload_data(const ObLobCommon *lob_comm, const ObString
     }
     if (OB_SUCC(ret)) {
       uint32 disk_lob_header_len = sizeof(ObLobCommon);
+      if (lob_comm->in_row_) {
         disk_lob_header_len += lob_comm->is_init_ ? sizeof(ObLobData) : 0;
+      } else {
+        disk_lob_header_len = lob_comm->get_handle_size(0);
+      }
       OB_ASSERT(payload.length() + disk_lob_header_len <= buf_len);
       MEMCPY(buf, lob_comm, disk_lob_header_len);
       if (payload.length() > 0) {

deps/oblib/src/common/object/ob_object.h

@@ -1496,7 +1496,7 @@ public:
       ObLobLocatorV2 loc(reinterpret_cast<char *>(v_.ptr_), val_len_, has_lob_header());
       if (OB_UNLIKELY(!loc.is_valid(false))) {
         inrow_data.assign_ptr(v_.string_, val_len_);
-      } else if (!loc.is_inrow()) {
+      } else if (!loc.has_inrow_data()) {
         inrow_data.assign_ptr("outrow", 6);
       } else if (OB_FAIL(loc.get_inrow_data(inrow_data))) {
         COMMON_LOG(WARN, "Lob: get inrow data failed in obobj", K(*this));
@@ -1519,7 +1519,7 @@ public:
       if (OB_UNLIKELY(!loc.is_valid(false))) {
         // do nothing, warn log inside
         COMMON_LOG(WARN, "Lob: invalid json lob", K(ret), K(json_data));
-      } else if (!loc.is_inrow()) {
+      } else if (!loc.has_inrow_data()) {
         if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s", "'outrow json'"))) {
           COMMON_LOG(WARN, "Lob: fail to print \"\'outrow json\'\"", K(ret), K(buf_len), K(pos));
         }
@@ -3045,7 +3045,7 @@ inline bool ObObj::is_outrow_lob() const
     } else {
       ObLobLocatorV2 loc(reinterpret_cast<char *>(v_.ptr_), val_len_, has_lob_header());
       if (loc.is_valid()) {
-        bret = !loc.is_inrow();
+        bret = !loc.has_inrow_data();
       }
     }
   }

src/share/ob_lob_access_utils.cpp

@@ -86,7 +86,7 @@ int ObTextStringIter::init(uint32_t buffer_len,
     } else if (!locator.is_valid()) {
       ret = OB_ERR_UNEXPECTED;
       COMMON_LOG(WARN,"Lob: invalid lob", K(ret));
-    } else if (FALSE_IT(is_outrow_ = !locator.is_inrow())) {
+    } else if (FALSE_IT(is_outrow_ = !locator.has_inrow_data())) {
     } else if (!is_outrow_) { // inrow lob always get full data, no need ctx_
     } else if (OB_ISNULL(allocator)) {
       ret = OB_INVALID_ARGUMENT;

src/storage/lob/ob_lob_locator.cpp

@@ -452,9 +452,15 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator,
     const ObLobCommon *lob_common =
       (payload.length() == 0 ? NULL : reinterpret_cast<const ObLobCommon *>(payload.ptr()));
     int64_t out_payload_len = payload.length();
+    int64_t byte_size = lob_common->get_byte_size(out_payload_len);
     bool is_src_inrow = (is_simple ? true : lob_common->in_row_);
     // systable read always get full lob data and output inrow lobs
-    bool is_dst_inrow = (is_systable ? true : is_src_inrow);
+    bool is_dst_inrow = ((is_systable) ? true : is_src_inrow);
+    if (byte_size <= LOB_FORCE_INROW_SIZE) {
+      // if lob is smaller than datum allow size
+      // let lob obj force inrow for hash/cmp cannot handle error
+      is_dst_inrow = true;
+    }
     // oracle user table lobs and mysql user table outrow lobs need extern.
     bool has_extern = (!is_simple) && (lib::is_oracle_mode() || !is_dst_inrow);
     ObMemLobExternFlags extern_flags(has_extern);
@@ -462,16 +468,18 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator,
     if (!is_src_inrow && is_dst_inrow) {
       // read outrow lobs but output as inrow lobs, need to calc the output payload lens
       // get byte size of out row lob, and calc total disk lob handle size if it is inrow
-      out_payload_len = lob_common->get_byte_size(out_payload_len);
+      out_payload_len += byte_size; // need whole disk locator
-      out_payload_len = ObLobCommon::calc_inrow_handle_size(lob_common->is_init_, out_payload_len);
     }
 
     int64_t full_loc_size = ObLobLocatorV2::calc_locator_full_len(extern_flags,
                                                                   rowid_str.length(),
                                                                   out_payload_len,
                                                                   is_simple);
-
+    if (full_loc_size > OB_MAX_LONGTEXT_LENGTH) {
-    if (OB_ISNULL(buf = reinterpret_cast<char *>(locator_allocator_.alloc(full_loc_size)))) {
+      ret = OB_SIZE_OVERFLOW;
+      STORAGE_LOG(WARN, "Failed to get lob data over size", K(ret), K(full_loc_size),
+                  K(rowid_str.length()), K(out_payload_len));
+    } else if (OB_ISNULL(buf = reinterpret_cast<char *>(locator_allocator_.alloc(full_loc_size)))) {
       ret = OB_ALLOCATE_MEMORY_FAILED;
       STORAGE_LOG(WARN, "Failed to alloc memory for lob locator", K(ret), K(full_loc_size));
     } else if (FALSE_IT(MEMSET(buf, 0, full_loc_size))) {
@@ -530,14 +538,8 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator,
             STORAGE_LOG(WARN, "Lob: get disk locator failed", K(ret), K(column_id));
           } else {
             char *buffer = disk_loc_str.ptr();
-            MEMCPY(buffer, lob_common, sizeof(ObLobCommon));
+            MEMCPY(buffer, lob_common, payload.length());
-            int64_t offset = sizeof(ObLobCommon);
+            int64_t offset = payload.length();
-            if (lob_common->is_init_) {
-              MEMCPY(buffer + offset, lob_common->buffer_, sizeof(ObLobData));
-              offset += sizeof(ObLobData); // copy lob id
-            }
-            ObLobCommon *new_lob_common = reinterpret_cast<ObLobCommon *>(buffer);
-            new_lob_common->in_row_ = 1; // set to inrow
 
             // read full data to new locator
             storage::ObLobManager* lob_mngr = MTL(storage::ObLobManager*);
@@ -561,7 +563,7 @@ int ObLobLocatorHelper::build_lob_locatorv2(ObLobLocatorV2 &locator,
             param.timeout_ = access_ctx.timeout_;
             param.scan_backward_ = false;
             param.offset_ = 0;
-            param.len_ = (disk_loc_str.length() - offset);
+            param.len_ = param.byte_size_;
             ObString output_data;
             output_data.assign_buffer(buffer + offset, param.len_);
             if (OB_FAIL(lob_mngr->query(param, output_data))) {

src/storage/lob/ob_lob_locator.h

@@ -57,6 +57,7 @@ public:
                KPC(rowid_project_), K_(rowid_objs), K_(enable_locator_v2), K_(is_inited));
 private:
   static const int64_t DEFAULT_LOCATOR_OBJ_ARRAY_SIZE = 8;
+  static const int64_t LOB_FORCE_INROW_SIZE = 64 * 1024L; // 64K
   int init_rowid_version(const share::schema::ObTableSchema &table_schema);
   int build_rowid_obj(blocksstable::ObDatumRow &row,
                       common::ObString &rowid_str,

src/storage/lob/ob_lob_manager.cpp

@@ -513,14 +513,20 @@ int ObLobManager::query(
       LOG_WARN("get lob data null.", K(ret));
     } else if (OB_FAIL(check_handle_size(param))) {
       LOG_WARN("check handle size failed.", K(ret));
-    } else if (lob_common->in_row_) {
+    } else if (lob_common->in_row_ || (param.lob_locator_ != nullptr && param.lob_locator_->has_inrow_data())) {
       ObString data;
+      if (param.lob_locator_ != nullptr && param.lob_locator_->has_inrow_data()) {
+        if (OB_FAIL(param.lob_locator_->get_inrow_data(data))) {
+          LOG_WARN("fail to get inrow data", K(ret), KPC(param.lob_locator_));
+        }
+      } else { // lob_common->in_row_
         if (lob_common->is_init_) {
           param.lob_data_ = reinterpret_cast<ObLobData*>(lob_common->buffer_);
           data.assign_ptr(param.lob_data_->buffer_, param.lob_data_->byte_size_);
         } else {
           data.assign_ptr(lob_common->buffer_, param.byte_size_);
         }
+      }
       uint32_t byte_offset = param.offset_ > data.length() ? data.length() : param.offset_;
       uint32_t max_len = ObCharset::strlen_char(param.coll_type_, data.ptr(), data.length()) - byte_offset;
       uint32_t byte_len = (param.len_ > max_len) ? max_len : param.len_;
@@ -644,6 +650,18 @@ int ObLobManager::query(
       LOG_WARN("get lob data null.", K(ret));
     } else if (OB_FAIL(check_handle_size(param))) {
       LOG_WARN("check handle size failed.", K(ret));
+    } else if (param.lob_locator_ != nullptr && param.lob_locator_->has_inrow_data()) {
+      ObString data;
+      if (OB_FAIL(param.lob_locator_->get_inrow_data(data))) {
+        LOG_WARN("fail to get inrow data", K(ret), KPC(param.lob_locator_));
+      } else if (OB_FAIL(query_inrow_get_iter(param, data, param.offset_, param.scan_backward_, result))) {
+        LOG_WARN("fail to get inrow query iter", K(ret));
+        if (OB_NOT_NULL(result)) {
+          result->reset();
+          common::sop_return(ObLobQueryIter, result);
+          result = nullptr;
+        }
+      }
     } else if (lob_common->in_row_) {
       ObString data;
       if (lob_common->is_init_) {
@@ -1107,7 +1125,7 @@ int ObLobManager::append(
   } else if (lob.is_delta_temp_lob()) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("invalid lob locator", K(ret));
-  } else if (lob.is_inrow()) {
+  } else if (lob.has_inrow_data()) {
     ObString data;
     if (OB_FAIL(lob.get_inrow_data(data))) {
       LOG_WARN("get inrow data int insert lob col failed", K(lob), K(data));
@@ -1911,17 +1929,26 @@ int ObLobManager::getlength(ObLobAccessParam& param, uint64_t &len)
       LOG_WARN("check handle size failed.", K(ret));
     } else if (!is_char) { // return byte len
       len = lob_common->get_byte_size(param.handle_size_);
-    } else if (lob_common->in_row_) { // calc char len
+    } else if (lob_handle_has_char_len(param)) {
+      len = *get_char_len_ptr(param);
+    } else if (lob_common->in_row_ || // calc char len
+               (param.lob_locator_ != nullptr && param.lob_locator_->has_inrow_data())) {
       ObString data;
+      if (param.lob_locator_ != nullptr && param.lob_locator_->has_inrow_data()) {
+        if (OB_FAIL(param.lob_locator_->get_inrow_data(data))) {
+          LOG_WARN("fail to get inrow data", K(ret), KPC(param.lob_locator_));
+        }
+      } else {
         if (lob_common->is_init_) {
           param.lob_data_ = reinterpret_cast<ObLobData*>(lob_common->buffer_);
           data.assign_ptr(param.lob_data_->buffer_, param.lob_data_->byte_size_);
         } else {
           data.assign_ptr(lob_common->buffer_, param.byte_size_);
         }
+      }
+      if (OB_SUCC(ret)) {
         len = ObCharset::strlen_char(param.coll_type_, data.ptr(), data.length());
-    } else if (lob_handle_has_char_len(param)) {
+      }
-      len = *get_char_len_ptr(param);
     } else { // do meta scan
       bool is_remote_lob = false;
       common::ObAddr dst_addr;
@@ -3152,7 +3179,7 @@ int ObLobManager::build_lob_param(ObLobAccessParam& param,
       param.len_ = len;
       param.timeout_ = timeout;
       // outrow arg for do lob meta scan
-      if (OB_SUCC(ret) && lob.is_persist_lob() && !lob.is_inrow()) {
+      if (OB_SUCC(ret) && lob.is_persist_lob() && !lob.has_inrow_data()) {
         ObMemLobTxInfo *tx_info = nullptr;
         ObMemLobLocationInfo *location_info = nullptr;
         if (OB_FAIL(lob.get_tx_info(tx_info))) {

src/storage/lob/ob_lob_manager.h

@@ -51,7 +51,7 @@ struct ObLobCompareParams {
   }
 
   TO_STRING_KV(K(collation_left_),
-               KP(collation_right_),
+               K(collation_right_),
                K(offset_left_),
                K(offset_right_),
                K(compare_len_),