database/openGauss-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

解决IVFPQ召回率问题

Browse Source
This commit is contained in:
wangjingyuan
2025-02-24 20:33:44 +08:00
committed by Lin_qiang
parent 29bd966ba4
commit a703a08fe3
7 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/common/backend/utils/adt/vector.cpp
View File

@ -1575,7 +1575,7 @@ struct ElementOpL2 {
struct ElementOpIP {
static float32x4_t op(float32x4_t x, float32x4_t y) {
return vsubq_f32(x, y);
return vmulq_f32(x, y);
}
};
@ -1590,9 +1590,7 @@ void VectorOpNYD4(size_t ny, float *x, char *pqTable, Size subSize, int offset,
for (i = 0; i < ny; i++) {
y = DatumGetVector(pqTable + (offset + i) * subSize)->x;
float32x4_t accu = ElementOp::op(x0, vld1q_f32(y));
accu = vaddq_f32(accu, accu);
accu = vaddq_f32(accu, accu);
dis[i] = vgetq_lane_f32(accu, 0);
dis[i] = vaddvq_f32(accu);
}
}
@ -1612,9 +1610,7 @@ void VectorOpNYD8(size_t ny, float *x, char *pqTable, Size subSize, int offset,
float32x4_t accu = ElementOp::op(x0, vld1q_f32(y));
y += batch;
accu = vaddq_f32(accu, ElementOp::op(x1, vld1q_f32(y)));
accu = vaddq_f32(accu, accu);
accu = vaddq_f32(accu, accu);
dis[i] = vgetq_lane_f32(accu, 0);
dis[i] = vaddvq_f32(accu);
}
}
@ -1640,9 +1636,7 @@ void VectorOpNYD16(size_t ny, float *x, char *pqTable, Size subSize, int offset,
accu = vaddq_f32(accu, ElementOp::op(x2, vld1q_f32(y)));
y += batch;
accu = vaddq_f32(accu, ElementOp::op(x3, vld1q_f32(y)));
accu = vaddq_f32(accu, accu);
accu = vaddq_f32(accu, accu);
dis[i] = vgetq_lane_f32(accu, 0);
dis[i] = vaddvq_f32(accu);
}
}
#endif
@ -1653,7 +1647,6 @@ void VectorL2SquaredDistanceNYRef(size_t d, size_t ny, float *x, char *pqTable,
for (size_t i = 0; i < ny; i++) {
y = DatumGetVector(pqTable + (offset + i) * subSize)->x;
dis[i] = VectorL2SquaredDistance(d, x, y);
y += d;
}
}

10
src/gausskernel/storage/access/datavec/ivfbuild.cpp
View File

@ -58,7 +58,7 @@ static void CreatePQPages(IvfflatBuildState *buildstate, ForkNumber fNum)
Buffer buf;
Page page;
uint16 pqTableNblk;
uint16 pqPreComputeTableNblk;
uint32 pqPreComputeTableNblk;
GenericXLogState *state;
IvfGetPQInfoFromMetaPage(index, &pqTableNblk, NULL, &pqPreComputeTableNblk, NULL);
@ -72,7 +72,7 @@ static void CreatePQPages(IvfflatBuildState *buildstate, ForkNumber fNum)
}
/* create pq distance table page */
for (uint16 i = 0; i < pqPreComputeTableNblk; i++) {
for (uint32 i = 0; i < pqPreComputeTableNblk; i++) {
buf = IvfflatNewBuffer(index, forkNum);
IvfflatInitRegisterPage(index, &buf, &page, &state);
MarkBufferDirty(buf);
@ -695,9 +695,9 @@ static void CreateMetaPage(Relation index, IvfflatBuildState *buildstate, ForkNu
(metap->pqTableSize + IVF_PQTABLE_STORAGE_SIZE - 1) / IVF_PQTABLE_STORAGE_SIZE);
if (buildstate->byResidual &&
(buildstate->params->funcType == IVF_PQ_DIS_L2 || buildstate->params->funcType == IVF_PQ_DIS_COSINE)) {
uint32 TableLen = buildstate->lists * buildstate->pqM * buildstate->pqKsub;
metap->pqPreComputeTableSize = (uint32)TableLen * sizeof(float);
metap->pqPreComputeTableNblk = (uint16)(
uint64 TableLen = buildstate->lists * buildstate->pqM * buildstate->pqKsub;
metap->pqPreComputeTableSize = (uint64)TableLen * sizeof(float);
metap->pqPreComputeTableNblk = (uint32)(
(metap->pqPreComputeTableSize + IVF_PQTABLE_STORAGE_SIZE - 1) / IVF_PQTABLE_STORAGE_SIZE);
}
} else {

2
src/gausskernel/storage/access/datavec/ivfinsert.cpp
View File

@ -37,7 +37,7 @@ static void FindInsertPage(Relation index, Datum *values, BlockNumber *insertPag
{
double minDistance = DBL_MAX;
uint16 pqTableNblk;
uint16 pqDisTableNblk;
uint32 pqDisTableNblk;
IvfGetPQInfoFromMetaPage(index, &pqTableNblk, NULL, &pqDisTableNblk, NULL);
BlockNumber nextblkno = IVFPQTABLE_START_BLKNO + pqTableNblk + pqDisTableNblk;
FmgrInfo *procinfo;

10
src/gausskernel/storage/access/datavec/ivfscan.cpp
View File

@ -54,7 +54,7 @@ static void GetScanLists(IndexScanDesc scan, Datum value)
{
IvfflatScanOpaque so = (IvfflatScanOpaque)scan->opaque;
uint16 pqTableNblk;
uint16 pqDisTableNblk;
uint32 pqDisTableNblk;
IvfGetPQInfoFromMetaPage(scan->indexRelation, &pqTableNblk, NULL, &pqDisTableNblk, NULL);
BlockNumber nextblkno = IVFPQTABLE_START_BLKNO + pqTableNblk + pqDisTableNblk;
int listId = 0;
@ -86,6 +86,12 @@ static void GetScanLists(IndexScanDesc scan, Datum value)
scanlist->distance = distance;
scanlist->key = listId;
listId++;
if (so->funcType == IVFPQ_DIS_COSINE && so->byResidual) {
Vector *vd = (Vector *)DatumGetPointer(value);
scanlist->pqDistance = VectorL2SquaredDistance(so->dimensions, list->center.x, vd->x);
} else {
scanlist->pqDistance = distance;
}
/* Add to heap */
pairingheap_add(so->listQueue, &scanlist->ph_node);
}
@ -246,7 +252,7 @@ static void GetScanItemsPQ(IndexScanDesc scan, Datum value, float *simTable)
int listCount = 0;
while (!pairingheap_is_empty(so->listQueue)) {
IvfflatScanList *scanlist = (IvfflatScanList *)pairingheap_remove_first(so->listQueue);
double dis0 = so->byResidual ? scanlist->distance : 0;
double dis0 = so->byResidual ? scanlist->pqDistance : 0;
BlockNumber searchPage = scanlist->startPage;
int key = scanlist->key;
float *simTable2;

22
src/gausskernel/storage/access/datavec/ivfutils.cpp
View File

@ -235,16 +235,16 @@ float* IVFPQLoadPQDisTable(Relation index)
Buffer buf;
Page page;
uint16 pqTableNblk;
uint16 nblks;
uint32 nblks;
uint32 curFlushSize;
uint32 pqDisTableSize;
uint64 pqDisTableSize;
float* disTable;
IvfGetPQInfoFromMetaPage(index, &pqTableNblk, NULL, &nblks, &pqDisTableSize);
disTable = (float*)palloc0(pqDisTableSize);
BlockNumber startBlkno = IVFPQTABLE_START_BLKNO + pqTableNblk;
for (uint16 i = 0; i < nblks; i++) {
for (uint32 i = 0; i < nblks; i++) {
curFlushSize = (i == nblks - 1) ? (pqDisTableSize - i * IVFPQTABLE_STORAGE_SIZE) : IVFPQTABLE_STORAGE_SIZE;
buf = ReadBuffer(index, startBlkno + i);
LockBuffer(buf, BUFFER_LOCK_SHARE);
@ -346,12 +346,12 @@ uint8 *LoadPQCode(IndexTuple itup)
float GetPQDistance(float *pqDistanceTable, uint8 *code, double dis0, int pqM, int pqKsub, bool innerPro)
{
float resDistance = dis0;
float resDistance = 0.0;
for (int i = 0; i < pqM; i++) {
int offset = i * pqKsub + code[i];
resDistance += pqDistanceTable[offset];
}
return innerPro ? (0 - resDistance) : resDistance;
return innerPro ? (dis0 - resDistance) : (dis0 + resDistance);
}
IvfpqPairingHeapNode * IvfpqCreatePairingHeapNode(float distance, ItemPointer heapTid,
@ -432,7 +432,7 @@ int getIVFPQfunctionType(FmgrInfo *procinfo, FmgrInfo *normprocinfo)
* Get the info related to pqTable in metapage
*/
void IvfGetPQInfoFromMetaPage(Relation index, uint16 *pqTableNblk, uint32 *pqTableSize,
uint16 *pqPreComputeTableNblk, uint32 *pqPreComputeTableSize)
uint32 *pqPreComputeTableNblk, uint64 *pqPreComputeTableSize)
{
Buffer buf;
Page page;
@ -528,14 +528,14 @@ int IvfGetByResidual(Relation index)
return IVFPQ_DEFAULT_RESIDUAL;
}
void IvfFlushPQInfoInternal(Relation index, char* table, BlockNumber startBlkno, uint16 nblks, uint32 totalSize)
void IvfFlushPQInfoInternal(Relation index, char* table, BlockNumber startBlkno, uint32 nblks, uint64 totalSize)
{
Buffer buf;
Page page;
uint32 curFlushSize;
GenericXLogState *state;
for (uint16 i = 0; i < nblks; i++) {
for (uint32 i = 0; i < nblks; i++) {
curFlushSize = (i == nblks - 1) ?
(totalSize - i * IVF_PQTABLE_STORAGE_SIZE) : IVF_PQTABLE_STORAGE_SIZE;
buf = ReadBufferExtended(index, MAIN_FORKNUM, startBlkno + i, RBM_NORMAL, NULL);
@ -560,14 +560,14 @@ void IvfFlushPQInfo(IvfflatBuildState *buildstate)
float* preComputeTable = buildstate->preComputeTable;
uint16 pqTableNblk;
uint32 pqTableSize;
uint16 pqPrecomputeTableNblk;
uint32 pqPrecomputeTableSize;
uint32 pqPrecomputeTableNblk;
uint64 pqPrecomputeTableSize;
IvfGetPQInfoFromMetaPage(index, &pqTableNblk, &pqTableSize, &pqPrecomputeTableNblk, &pqPrecomputeTableSize);
/* Flush pq table */
IvfFlushPQInfoInternal(index, pqTable, IVF_PQTABLE_START_BLKNO, pqTableNblk, pqTableSize);
if (buildstate->byResidual && buildstate->params->funcType == IVF_PQ_DIS_L2) {
if (buildstate->byResidual && buildstate->params->funcType != IVF_PQ_DIS_IP) {
/* Flush pq distance table */
IvfFlushPQInfoInternal(index, (char*)preComputeTable,
IVF_PQTABLE_START_BLKNO + pqTableNblk, pqPrecomputeTableNblk, pqPrecomputeTableSize);

2
src/gausskernel/storage/access/datavec/ivfvacuum.cpp
View File

@ -34,7 +34,7 @@ IndexBulkDeleteResult *ivfflatbulkdelete_internal(IndexVacuumInfo *info, IndexBu
IndexBulkDeleteCallback callback, void *callbackState)
{
uint16 pqTableNblk;
uint16 pqDisTableNblk;
uint32 pqDisTableNblk;
Relation index = info->index;
IvfGetPQInfoFromMetaPage(index, &pqTableNblk, NULL, &pqDisTableNblk, NULL);
BlockNumber blkno = IVFPQTABLE_START_BLKNO + pqTableNblk + pqDisTableNblk;

7
src/include/access/datavec/ivfflat.h
View File

@ -261,7 +261,7 @@ typedef struct IvfflatMetaPageData {
uint32 pqTableSize;
uint16 pqTableNblk;
uint64 pqPreComputeTableSize;
uint16 pqPreComputeTableNblk;
uint32 pqPreComputeTableNblk;
} IvfflatMetaPageData;
typedef IvfflatMetaPageData *IvfflatMetaPage;
@ -287,6 +287,7 @@ typedef struct IvfflatScanList {
BlockNumber startPage;
double distance;
int key;
double pqDistance;
} IvfflatScanList;
typedef struct IvfflatScanOpaqueData {
@ -356,9 +357,9 @@ int IvfGetByResidual(Relation index);
void IvfGetPQInfoFromMetaPage(Relation index, uint16 *pqTableNblk, uint32 *pqTableSize,
uint16 *pqPreComputeTableNblk, uint32 *pqPreComputeTableSize);
uint32 *pqPreComputeTableNblk, uint64 *pqPreComputeTableSize);
int getIVFPQfunctionType(FmgrInfo *procinfo, FmgrInfo *normprocinfo);
void IvfFlushPQInfoInternal(Relation index, char* table, BlockNumber startBlkno, uint16 nblks, uint32 totalSize);
void IvfFlushPQInfoInternal(Relation index, char* table, BlockNumber startBlkno, uint32 nblks, uint64 totalSize);
void IvfFlushPQInfo(IvfflatBuildState *buildstate);
int IvfComputePQTable(VectorArray samples, PQParams *params);