liuyu/pgvector
1
0
Fork 0
mirror of https://github.com/pgvector/pgvector.git synced 2026-06-30 09:41:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improved memory estimate for HNSW index builds

Browse Source
This commit is contained in:
Andrew Kane
2024-01-03 13:47:50 -05:00
parent 9b73b3d1a6
commit 8ee37b60a0
3 changed files with 9 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/hnsw.h
View File

@@ -330,7 +330,6 @@ List *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, Fmgr
HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
HnswElement HnswInitElement(ItemPointer tid, int m, double ml, int maxLevel);
void HnswFreeElement(HnswElement element);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswElement HnswFindDuplicate(HnswElement e);

27
src/hnswbuild.c
View File

@@ -310,11 +310,9 @@ HnswElementMemory(HnswElement e, int m)
elementSize += sizeof(HnswNeighborArray) * (e->level + 1);
elementSize += sizeof(HnswCandidate) * (m * (e->level + 2));
elementSize += sizeof(List);
elementSize += sizeof(ItemPointerData) + SIZEOF_VOID_P;
elementSize += VARSIZE_ANY(DatumGetPointer(e->value));
/* Each allocation has a chunk header */
elementSize += (e->level + 7) * GENERATIONCHUNK_RAWSIZE;
elementSize += (e->level + 4) * GENERATIONCHUNK_RAWSIZE;
/* Add an extra 5% for alignment and other overhead */
return elementSize * 1.05;
}
@@ -379,28 +377,21 @@ InsertTupleInMemory(Relation index, Datum *values, ItemPointer heaptid, HnswBuil
if (dup != NULL)
{
/* No need to free element since memory unlikely to be reallocated */
HnswAddHeapTid(dup, heaptid);
HnswFreeElement(element);
#if PG_VERSION_NUM >= 130000
buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->memoryUsed += sizeof(ItemPointerData) + SIZEOF_VOID_P + GENERATIONCHUNK_RAWSIZE;
#endif
}
else
{
buildstate->elements = lappend(buildstate->elements, element);
#if PG_VERSION_NUM >= 130000
buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->memoryUsed += HnswElementMemory(element, buildstate->m);
#endif
}
MemoryContextSwitchTo(oldCtx);
/* Update memory usage */
#if PG_VERSION_NUM >= 130000
buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->memoryUsed += HnswElementMemory(element, buildstate->m);
#endif
return true;
}

23
src/hnswutils.c
View File

@@ -206,17 +206,6 @@ HnswInitNeighbors(HnswElement element, int m)
}
}
/*
* Free neighbors
*/
static void
HnswFreeNeighbors(HnswElement element)
{
for (int lc = 0; lc <= element->level; lc++)
pfree(element->neighbors[lc].items);
pfree(element->neighbors);
}
/*
* Allocate an element
*/
@@ -244,18 +233,6 @@ HnswInitElement(ItemPointer heaptid, int m, double ml, int maxLevel)
return element;
}
/*
* Free an element
*/
void
HnswFreeElement(HnswElement element)
{
HnswFreeNeighbors(element);
if (DatumGetPointer(element->value))
pfree(DatumGetPointer(element->value));
pfree(element);
}
/*
* Add a heap TID to an element
*/