liuyu/pgvector
1
0
Fork 0
mirror of https://github.com/pgvector/pgvector.git synced 2026-07-11 23:26:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improved memory estimate for HNSW index builds

Browse Source
This commit is contained in:
Andrew Kane
2024-01-03 13:47:50 -05:00
parent 9b73b3d1a6
commit 8ee37b60a0
3 changed files with 9 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/hnsw.h
View File

@@ -330,7 +330,6 @@ List *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, Fmgr
HnswElement HnswGetEntryPoint(Relation index); HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint); void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
HnswElement HnswInitElement(ItemPointer tid, int m, double ml, int maxLevel); HnswElement HnswInitElement(ItemPointer tid, int m, double ml, int maxLevel);
void HnswFreeElement(HnswElement element);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno); HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing); void HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswElement HnswFindDuplicate(HnswElement e); HnswElement HnswFindDuplicate(HnswElement e);

19
src/hnswbuild.c
View File

@@ -310,11 +310,9 @@ HnswElementMemory(HnswElement e, int m)
elementSize += sizeof(HnswNeighborArray) * (e->level + 1); elementSize += sizeof(HnswNeighborArray) * (e->level + 1);
elementSize += sizeof(HnswCandidate) * (m * (e->level + 2)); elementSize += sizeof(HnswCandidate) * (m * (e->level + 2));
elementSize += sizeof(List);
elementSize += sizeof(ItemPointerData) + SIZEOF_VOID_P;
elementSize += VARSIZE_ANY(DatumGetPointer(e->value)); elementSize += VARSIZE_ANY(DatumGetPointer(e->value));
/* Each allocation has a chunk header */ /* Each allocation has a chunk header */
elementSize += (e->level + 7) * GENERATIONCHUNK_RAWSIZE; elementSize += (e->level + 4) * GENERATIONCHUNK_RAWSIZE;
/* Add an extra 5% for alignment and other overhead */ /* Add an extra 5% for alignment and other overhead */
return elementSize * 1.05; return elementSize * 1.05;
} }
@@ -379,27 +377,20 @@ InsertTupleInMemory(Relation index, Datum *values, ItemPointer heaptid, HnswBuil
if (dup != NULL) if (dup != NULL)
{ {
/* No need to free element since memory unlikely to be reallocated */
HnswAddHeapTid(dup, heaptid); HnswAddHeapTid(dup, heaptid);
HnswFreeElement(element);
#if PG_VERSION_NUM >= 130000
buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->memoryUsed += sizeof(ItemPointerData) + SIZEOF_VOID_P + GENERATIONCHUNK_RAWSIZE;
#endif
} }
else else
{
buildstate->elements = lappend(buildstate->elements, element); buildstate->elements = lappend(buildstate->elements, element);
MemoryContextSwitchTo(oldCtx);
/* Update memory usage */
#if PG_VERSION_NUM >= 130000 #if PG_VERSION_NUM >= 130000
buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false); buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else #else
buildstate->memoryUsed += HnswElementMemory(element, buildstate->m); buildstate->memoryUsed += HnswElementMemory(element, buildstate->m);
#endif #endif
}
MemoryContextSwitchTo(oldCtx);
return true; return true;
} }

23
src/hnswutils.c
View File

@@ -206,17 +206,6 @@ HnswInitNeighbors(HnswElement element, int m)
} }
} }
/*
* Free neighbors
*/
static void
HnswFreeNeighbors(HnswElement element)
{
for (int lc = 0; lc <= element->level; lc++)
pfree(element->neighbors[lc].items);
pfree(element->neighbors);
}
/* /*
* Allocate an element * Allocate an element
*/ */
@@ -244,18 +233,6 @@ HnswInitElement(ItemPointer heaptid, int m, double ml, int maxLevel)
return element; return element;
} }
/*
* Free an element
*/
void
HnswFreeElement(HnswElement element)
{
HnswFreeNeighbors(element);
if (DatumGetPointer(element->value))
pfree(DatumGetPointer(element->value));
pfree(element);
}
/* /*
* Add a heap TID to an element * Add a heap TID to an element
*/ */