hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vector index] fix vsag alloc failed case coredump

Browse Source
This commit is contained in:
Carrot-77 2024-09-26 08:16:45 +00:00 committed by ob-robot
parent 09f783dd0e
commit e2ca6caa6a
11 changed files with 32 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/init/oceanbase.el7.aarch64.deps vendored
View File

@ -34,7 +34,7 @@ devdeps-protobuf-c-1.4.1-100000072023102410.el7.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.aarch64.rpm
devdeps-apache-arrow-9.0.0-302024052920.el7.aarch64.rpm
devdeps-apache-orc-1.8.3-392024091317.el7.aarch64.rpm
devdeps-vsag-1.0.0-102024082710.el7.aarch64.rpm
devdeps-vsag-1.0.0-212024092509.el7.aarch64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm

2
deps/init/oceanbase.el7.x86_64.deps vendored
View File

@ -38,7 +38,7 @@ devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.x86_64.rpm
devdeps-apache-arrow-9.0.0-222024052223.el7.x86_64.rpm
devdeps-hyperscan-5.4.2-242024090219.el7.x86_64.rpm
devdeps-apache-orc-1.8.3-392024091317.el7.x86_64.rpm
devdeps-vsag-1.0.0-102024082710.el7.x86_64.rpm
devdeps-vsag-1.0.0-212024092509.el7.x86_64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm

2
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -34,7 +34,7 @@ devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
devdeps-apache-orc-1.8.3-392024091317.el8.aarch64.rpm
devdeps-vsag-1.0.0-102024082710.el8.aarch64.rpm
devdeps-vsag-1.0.0-212024092509.el8.aarch64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -36,7 +36,7 @@ devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm
devdeps-hyperscan-5.4.2-242024090219.el8.x86_64.rpm
devdeps-apache-orc-1.8.3-392024091317.el8.x86_64.rpm
devdeps-vsag-1.0.0-102024082710.el8.x86_64.rpm
devdeps-vsag-1.0.0-212024092509.el8.x86_64.rpm
devdeps-cloud-qpl-1.1.0-272023061419.el8.x86_64.rpm
[tools]

2
deps/init/oceanbase.el9.aarch64.deps vendored
View File

@ -38,7 +38,7 @@ devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
devdeps-apache-orc-1.8.3-392024091317.el8.aarch64.rpm
devdeps-vsag-1.0.0-102024082710.el8.aarch64.rpm
devdeps-vsag-1.0.0-212024092509.el8.aarch64.rpm
[deps-el9]
devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9

2
deps/init/oceanbase.el9.x86_64.deps vendored
View File

@ -41,7 +41,7 @@ devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
devdeps-hyperscan-5.4.2-242024090219.el8.x86_64.rpm
devdeps-apache-orc-1.8.3-392024091317.el8.x86_64.rpm
devdeps-vsag-1.0.0-102024082710.el8.x86_64.rpm
devdeps-vsag-1.0.0-212024092509.el8.x86_64.rpm
[deps-el9]
devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9

2
deps/oblib/src/lib/CMakeLists.txt vendored
View File

@ -431,7 +431,7 @@ if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
libdiskann.a
libopenblas.a
libgfortran_static.a
libquadmath.a
libquadmath_static.a
libgomp_static.a
libroaring.a
)

20
src/share/vector_index/ob_plugin_vector_index_adaptor.cpp
View File

@ -637,7 +637,7 @@ int ObPluginVectorIndexAdaptor::init_mem_data(ObVectorIndexRecordType type)
param->ef_construction_,
param->ef_search_,
incr_data_->mem_ctx_))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("failed to create vsag index.", K(ret));
}
}
@ -713,7 +713,7 @@ int ObPluginVectorIndexAdaptor::init_mem_data(ObVectorIndexRecordType type)
param->ef_construction_,
param->ef_search_,
snap_data_->mem_ctx_))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("failed to create vsag index.", K(ret), K(snap_data_->index_), KPC(param));
}
}
@ -890,7 +890,7 @@ int ObPluginVectorIndexAdaptor::insert_rows(blocksstable::ObDatumRow *rows,
incr_vids,
dim,
incr_vid_count))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("failed to add index.", K(ret), K(dim), K(row_count));
} else {
incr_data_->curr_vid_max_ = incr_data_->curr_vid_max_ > max_vid ? incr_data_->curr_vid_max_ : max_vid;
@ -939,7 +939,7 @@ int ObPluginVectorIndexAdaptor::add_snap_index(float *vectors, int64_t *vids, in
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(tenant_id_, "VIndexVsagADP"));
TCWLockGuard lock_guard(snap_data_->mem_data_rwlock_);
if (OB_FAIL(obvectorutil::add_index(snap_data_->index_, vectors, vids, dim, num))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("failed to build index.", K(ret), K(dim), K(num));
}
}
@ -1042,7 +1042,7 @@ int ObPluginVectorIndexAdaptor::write_into_delta_mem(ObVectorQueryAdaptorResultC
reinterpret_cast<int64_t *>(vids),
ctx->get_dim(),
count))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("failed to add index.", K(ret), K(ctx->get_dim()), K(count));
}
LOG_TRACE("write into delta mem.", K(ret), K(ctx->get_dim()), K(count));
@ -1441,7 +1441,7 @@ int ObPluginVectorIndexAdaptor::serialize(ObIAllocator *allocator, ObOStreamBuf:
ret = OB_ERR_UNEXPECTED;
LOG_WARN("snap index is not init", K(ret));
} else if (OB_FAIL(obvectorutil::get_index_number(snap_data_->index_, snap_index_size))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("failed to get snap index number.", K(ret));
} else if (snap_index_size == 0) {
// do nothing
@ -1636,7 +1636,7 @@ int ObPluginVectorIndexAdaptor::vsag_query_vids(ObVectorQueryAdaptorResultContex
delta_res_cnt,
query_cond->ef_search_,
ibitmap))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("knn search delta failed.", K(ret), K(dim));
}
}
@ -1653,7 +1653,7 @@ int ObPluginVectorIndexAdaptor::vsag_query_vids(ObVectorQueryAdaptorResultContex
snap_res_cnt,
query_cond->ef_search_,
dbitmap))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("knn search snap failed.", K(ret), K(dim));
}
}
@ -2099,7 +2099,7 @@ int ObPluginVectorIndexAdaptor::check_need_sync_to_follower(bool &need_sync)
int64_t current_incr_count = 0;
if (OB_NOT_NULL(get_incr_index())) {
if (OB_FAIL(obvectorutil::get_index_number(get_incr_index(), current_incr_count))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("fail to get incr index number", K(ret));
ret = OB_SUCCESS; // continue to check other parts
}
@ -2119,7 +2119,7 @@ int ObPluginVectorIndexAdaptor::check_need_sync_to_follower(bool &need_sync)
int64_t current_snapshot_count = 0;
if (OB_NOT_NULL(get_snap_index())) {
if (OB_FAIL(obvectorutil::get_index_number(get_snap_index(), current_snapshot_count))) {
ret = OB_ERR_VSAG_RETURN_ERROR;
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("fail to get snap index number", K(ret));
ret = OB_SUCCESS; // continue to check other parts
}

2
src/share/vector_index/ob_plugin_vector_index_serialize.cpp
View File

@ -192,6 +192,7 @@ int ObVectorIndexSerializer::serialize(void *index, ObOStreamBuf::CbParam &cb_pa
std::ostream out(&streambuf);
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(tenant_id, "VIndexVsagADP"));
if (OB_FAIL(obvectorutil::fserialize(index, out))) {
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("fail to do vsag serialize", K(ret));
} else {
streambuf.check_finish(); // do last callback to ensure all the data is written
@ -223,6 +224,7 @@ int ObVectorIndexSerializer::deserialize(void *&index, ObIStreamBuf::CbParam &cb
} else {
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(tenant_id, "VIndexVsagADP"));
if (OB_FAIL(obvectorutil::fdeserialize(index, in))) {
ret = ObPluginVectorIndexHelper::vsag_errcode_2ob(ret);
LOG_WARN("fail to do vsag deserialize", K(ret));
}
}

12
src/share/vector_index/ob_plugin_vector_index_util.cpp
View File

@ -240,5 +240,17 @@ int ObPluginVectorIndexHelper::get_vector_memory_limit_size(const uint64_t tenan
}
int ObPluginVectorIndexHelper::vsag_errcode_2ob(int vsag_errcode)
{
int ret = OB_ERR_VSAG_RETURN_ERROR;
if (vsag_errcode == 10) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("vsag failed to allocator.", K(ret), K(vsag_errcode));
} else {
LOG_WARN("get vsag failed.", K(ret), K(vsag_errcode));
}
return ret;
}
};
};

1
src/share/vector_index/ob_plugin_vector_index_util.h
View File

@ -91,6 +91,7 @@ public:
static int get_vector_memory_limit_size(const uint64_t tenant_id,
int64_t& memory_limit);
static int vsag_errcode_2ob(int vsag_errcode);
};
};