diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index b4be3a8bb4..aa8dc52f3c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -247,8 +247,6 @@ public:
 
     Status add_array_values(size_t field_size, const CollectionValue* values,
                             size_t count) override {
-        auto* item_data_ptr = const_cast<CollectionValue*>(values)->mutable_data();
-
         if constexpr (field_is_slice_type(field_type)) {
             if (_field == nullptr || _index_writer == nullptr) {
                 LOG(ERROR) << "field or index writer is null in inverted index writer.";
@@ -256,6 +254,7 @@ public:
                         "field or index writer is null in inverted index writer");
             }
             for (int i = 0; i < count; ++i) {
+                auto* item_data_ptr = const_cast<CollectionValue*>(values)->mutable_data();
                 std::vector<std::string> strings;
 
                 for (size_t j = 0; j < values->length(); ++j) {
@@ -270,11 +269,14 @@ public:
                 new_fulltext_field(value.c_str(), value.length());
                 _rid++;
                 _index_writer->addDocument(_doc);
+                values++;
             }
         } else if constexpr (field_is_numeric_type(field_type)) {
-            auto p = reinterpret_cast<const CppType*>(item_data_ptr);
             for (int i = 0; i < count; ++i) {
+                auto* item_data_ptr = const_cast<CollectionValue*>(values)->mutable_data();
+
                 for (size_t j = 0; j < values->length(); ++j) {
+                    const CppType* p = reinterpret_cast<const CppType*>(item_data_ptr);
                     if (values->is_null_at(j)) {
                         // bkd do not index null values, so we do nothing here.
                     } else {
@@ -284,10 +286,11 @@ public:
                         _value_key_coder->full_encode_ascending(p, &new_value);
                         _bkd_writer->add((const uint8_t*)new_value.c_str(), value_length, _rid);
                     }
-                    p++;
+                    item_data_ptr = (uint8_t*)item_data_ptr + field_size;
                 }
                 _row_ids_seen_for_bkd++;
                 _rid++;
+                values++;
             }
         }
         return Status::OK();
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index c2b65fed34..af254038bc 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -731,7 +731,6 @@ Status SchemaChangeForInvertedIndex::_add_nullable(
             auto step = next_run_step();
             if (null_map[offset]) {
                 RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_nulls(step));
-                *ptr += field->size() * step;
             } else {
                 if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
                     DCHECK(field->get_sub_field_count() == 1);
@@ -743,6 +742,7 @@ Status SchemaChangeForInvertedIndex::_add_nullable(
                             column_name, *ptr, step));
                 }
             }
+            *ptr += field->size() * step;
             offset += step;
         } while (offset < num_rows);
     } catch (const std::exception& e) {