Reduced WAL generation for HNSW index builds - thanks @hlinnaka

This commit is contained in:
Andrew Kane
2023-12-19 20:37:32 -05:00
parent 921427ee03
commit e88a425c9b
6 changed files with 132 additions and 86 deletions

View File

@@ -2,6 +2,7 @@
- Improved performance of HNSW - Improved performance of HNSW
- Added support for on-disk parallel index builds for HNSW - Added support for on-disk parallel index builds for HNSW
- Reduced WAL generation for HNSW index builds
- Fixed `invalid memory alloc request size` error with HNSW index build - Fixed `invalid memory alloc request size` error with HNSW index build
## 0.5.1 (2023-10-10) ## 0.5.1 (2023-10-10)

View File

@@ -318,10 +318,8 @@ int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index); int HnswGetEfConstruction(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum); FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result); bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
void HnswCommitBuffer(Buffer buf, GenericXLogState *state);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum); Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page); void HnswInitPage(Buffer buf, Page page);
void HnswInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state);
void HnswInit(void); void HnswInit(void);
List *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement); List *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
HnswElement HnswGetEntryPoint(Relation index); HnswElement HnswGetEntryPoint(Relation index);
@@ -332,12 +330,12 @@ HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing); void HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswElement HnswFindDuplicate(HnswElement e); HnswElement HnswFindDuplicate(HnswElement e);
HnswCandidate *HnswEntryCandidate(HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec); HnswCandidate *HnswEntryCandidate(HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum); void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(HnswNeighborTuple ntup, HnswElement e, int m); void HnswSetNeighborTuple(HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid); void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
void HnswInitNeighbors(HnswElement element, int m); void HnswInitNeighbors(HnswElement element, int m);
bool HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel); bool HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel, bool building);
void HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting); void HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec); void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec); void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswSetElementTuple(HnswElementTuple etup, HnswElement element); void HnswSetElementTuple(HnswElementTuple etup, HnswElement element);

View File

@@ -68,11 +68,11 @@ CreateMetaPage(HnswBuildState * buildstate)
ForkNumber forkNum = buildstate->forkNum; ForkNumber forkNum = buildstate->forkNum;
Buffer buf; Buffer buf;
Page page; Page page;
GenericXLogState *state;
HnswMetaPage metap; HnswMetaPage metap;
buf = HnswNewBuffer(index, forkNum); buf = HnswNewBuffer(index, forkNum);
HnswInitRegisterPage(index, &buf, &page, &state); page = BufferGetPage(buf);
HnswInitPage(buf, page);
/* Set metapage data */ /* Set metapage data */
metap = HnswPageGetMeta(page); metap = HnswPageGetMeta(page);
@@ -88,14 +88,14 @@ CreateMetaPage(HnswBuildState * buildstate)
((PageHeader) page)->pd_lower = ((PageHeader) page)->pd_lower =
((char *) metap + sizeof(HnswMetaPageData)) - (char *) page; ((char *) metap + sizeof(HnswMetaPageData)) - (char *) page;
HnswCommitBuffer(buf, state); UnlockReleaseBuffer(buf);
} }
/* /*
* Add a new page * Add a new page
*/ */
static void static void
HnswBuildAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state, ForkNumber forkNum) HnswBuildAppendPage(Relation index, Buffer *buf, Page *page, ForkNumber forkNum)
{ {
/* Add a new page */ /* Add a new page */
Buffer newbuf = HnswNewBuffer(index, forkNum); Buffer newbuf = HnswNewBuffer(index, forkNum);
@@ -104,7 +104,6 @@ HnswBuildAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **
HnswPageGetOpaque(*page)->nextblkno = BufferGetBlockNumber(newbuf); HnswPageGetOpaque(*page)->nextblkno = BufferGetBlockNumber(newbuf);
/* Commit */ /* Commit */
GenericXLogFinish(*state);
UnlockReleaseBuffer(*buf); UnlockReleaseBuffer(*buf);
/* Can take a while, so ensure we can interrupt */ /* Can take a while, so ensure we can interrupt */
@@ -115,8 +114,7 @@ HnswBuildAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **
/* Prepare new page */ /* Prepare new page */
*buf = newbuf; *buf = newbuf;
*state = GenericXLogStart(index); *page = BufferGetPage(*buf);
*page = GenericXLogRegisterBuffer(*state, *buf, GENERIC_XLOG_FULL_IMAGE);
HnswInitPage(*buf, *page); HnswInitPage(*buf, *page);
} }
@@ -135,7 +133,6 @@ CreateElementPages(HnswBuildState * buildstate)
BlockNumber insertPage; BlockNumber insertPage;
Buffer buf; Buffer buf;
Page page; Page page;
GenericXLogState *state;
ListCell *lc; ListCell *lc;
/* Calculate sizes */ /* Calculate sizes */
@@ -148,8 +145,7 @@ CreateElementPages(HnswBuildState * buildstate)
/* Prepare first page */ /* Prepare first page */
buf = HnswNewBuffer(index, forkNum); buf = HnswNewBuffer(index, forkNum);
state = GenericXLogStart(index); page = BufferGetPage(buf);
page = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE);
HnswInitPage(buf, page); HnswInitPage(buf, page);
foreach(lc, buildstate->elements) foreach(lc, buildstate->elements)
@@ -175,7 +171,7 @@ CreateElementPages(HnswBuildState * buildstate)
/* Keep element and neighbors on the same page if possible */ /* Keep element and neighbors on the same page if possible */
if (PageGetFreeSpace(page) < etupSize || (combinedSize <= maxSize && PageGetFreeSpace(page) < combinedSize)) if (PageGetFreeSpace(page) < etupSize || (combinedSize <= maxSize && PageGetFreeSpace(page) < combinedSize))
HnswBuildAppendPage(index, &buf, &page, &state, forkNum); HnswBuildAppendPage(index, &buf, &page, forkNum);
/* Calculate offsets */ /* Calculate offsets */
element->blkno = BufferGetBlockNumber(buf); element->blkno = BufferGetBlockNumber(buf);
@@ -199,7 +195,7 @@ CreateElementPages(HnswBuildState * buildstate)
/* Add new page if needed */ /* Add new page if needed */
if (PageGetFreeSpace(page) < ntupSize) if (PageGetFreeSpace(page) < ntupSize)
HnswBuildAppendPage(index, &buf, &page, &state, forkNum); HnswBuildAppendPage(index, &buf, &page, forkNum);
/* Add placeholder for neighbors */ /* Add placeholder for neighbors */
if (PageAddItem(page, (Item) ntup, ntupSize, InvalidOffsetNumber, false, false) != element->neighborOffno) if (PageAddItem(page, (Item) ntup, ntupSize, InvalidOffsetNumber, false, false) != element->neighborOffno)
@@ -209,10 +205,9 @@ CreateElementPages(HnswBuildState * buildstate)
insertPage = BufferGetBlockNumber(buf); insertPage = BufferGetBlockNumber(buf);
/* Commit */ /* Commit */
GenericXLogFinish(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_ALWAYS, buildstate->entryPoint, insertPage, forkNum); HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_ALWAYS, buildstate->entryPoint, insertPage, forkNum, true);
pfree(etup); pfree(etup);
pfree(ntup); pfree(ntup);
@@ -238,7 +233,6 @@ CreateNeighborPages(HnswBuildState * buildstate)
HnswElement e = lfirst(lc); HnswElement e = lfirst(lc);
Buffer buf; Buffer buf;
Page page; Page page;
GenericXLogState *state;
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m); Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m);
/* Can take a while, so ensure we can interrupt */ /* Can take a while, so ensure we can interrupt */
@@ -247,8 +241,7 @@ CreateNeighborPages(HnswBuildState * buildstate)
buf = ReadBufferExtended(index, forkNum, e->neighborPage, RBM_NORMAL, NULL); buf = ReadBufferExtended(index, forkNum, e->neighborPage, RBM_NORMAL, NULL);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
state = GenericXLogStart(index); page = BufferGetPage(buf);
page = GenericXLogRegisterBuffer(state, buf, 0);
HnswSetNeighborTuple(ntup, e, m); HnswSetNeighborTuple(ntup, e, m);
@@ -256,7 +249,6 @@ CreateNeighborPages(HnswBuildState * buildstate)
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index)); elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Commit */ /* Commit */
GenericXLogFinish(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
} }
@@ -398,7 +390,7 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx); oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
if (HnswInsertTuple(buildstate->index, values, isnull, tid, buildstate->heap)) if (HnswInsertTuple(buildstate->index, values, isnull, tid, buildstate->heap, true))
{ {
if (buildstate->hnswshared) if (buildstate->hnswshared)
{ {
@@ -880,6 +872,22 @@ BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
HnswEndParallel(buildstate->hnswleader); HnswEndParallel(buildstate->hnswleader);
} }
#if PG_VERSION_NUM < 110008
void
log_newpage_range(Relation rel, ForkNumber forkNum, BlockNumber startblk, BlockNumber endblk, bool page_std)
{
for (BlockNumber blkno = startblk; blkno < endblk; blkno++)
{
Buffer buf = ReadBufferExtended(rel, forkNum, blkno, RBM_NORMAL, NULL);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
MarkBufferDirty(buf);
log_newpage_buffer(buf, page_std);
UnlockReleaseBuffer(buf);
}
}
#endif
/* /*
* Build the index * Build the index
*/ */
@@ -895,6 +903,9 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
if (!buildstate->flushed) if (!buildstate->flushed)
FlushPages(buildstate); FlushPages(buildstate);
if (RelationNeedsWAL(index))
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocks(index), true);
FreeBuildState(buildstate); FreeBuildState(buildstate);
} }

View File

@@ -92,7 +92,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
* Add a new page * Add a new page
*/ */
static void static void
HnswInsertAppendPage(Relation index, Buffer *nbuf, Page *npage, GenericXLogState *state, Page page) HnswInsertAppendPage(Relation index, Buffer *nbuf, Page *npage, GenericXLogState *state, Page page, bool building)
{ {
/* Add a new page */ /* Add a new page */
LockRelationForExtension(index, ExclusiveLock); LockRelationForExtension(index, ExclusiveLock);
@@ -100,7 +100,11 @@ HnswInsertAppendPage(Relation index, Buffer *nbuf, Page *npage, GenericXLogState
UnlockRelationForExtension(index, ExclusiveLock); UnlockRelationForExtension(index, ExclusiveLock);
/* Init new page */ /* Init new page */
*npage = GenericXLogRegisterBuffer(state, *nbuf, GENERIC_XLOG_FULL_IMAGE); if (building)
*npage = BufferGetPage(*nbuf);
else
*npage = GenericXLogRegisterBuffer(state, *nbuf, GENERIC_XLOG_FULL_IMAGE);
HnswInitPage(*nbuf, *npage); HnswInitPage(*nbuf, *npage);
/* Update previous buffer */ /* Update previous buffer */
@@ -111,7 +115,7 @@ HnswInsertAppendPage(Relation index, Buffer *nbuf, Page *npage, GenericXLogState
* Add to element and neighbor pages * Add to element and neighbor pages
*/ */
static void static void
WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPage, BlockNumber *updatedInsertPage) WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPage, BlockNumber *updatedInsertPage, bool building)
{ {
Buffer buf; Buffer buf;
Page page; Page page;
@@ -151,8 +155,16 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
buf = ReadBuffer(index, currentPage); buf = ReadBuffer(index, currentPage);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
state = GenericXLogStart(index); if (building)
page = GenericXLogRegisterBuffer(state, buf, 0); {
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Keep track of first page where element at level 0 can fit */ /* Keep track of first page where element at level 0 can fit */
if (!BlockNumberIsValid(newInsertPage) && PageGetFreeSpace(page) >= minCombinedSize) if (!BlockNumberIsValid(newInsertPage) && PageGetFreeSpace(page) >= minCombinedSize)
@@ -172,7 +184,12 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage)) if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
{ {
if (nbuf != buf) if (nbuf != buf)
npage = GenericXLogRegisterBuffer(state, nbuf, 0); {
if (building)
npage = BufferGetPage(nbuf);
else
npage = GenericXLogRegisterBuffer(state, nbuf, 0);
}
break; break;
} }
@@ -181,7 +198,7 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
/* Skip if both tuples can fit on the same page */ /* Skip if both tuples can fit on the same page */
if (combinedSize > maxSize && PageGetFreeSpace(page) >= etupSize && !BlockNumberIsValid(HnswPageGetOpaque(page)->nextblkno)) if (combinedSize > maxSize && PageGetFreeSpace(page) >= etupSize && !BlockNumberIsValid(HnswPageGetOpaque(page)->nextblkno))
{ {
HnswInsertAppendPage(index, &nbuf, &npage, state, page); HnswInsertAppendPage(index, &nbuf, &npage, state, page, building);
break; break;
} }
@@ -190,7 +207,8 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
if (BlockNumberIsValid(currentPage)) if (BlockNumberIsValid(currentPage))
{ {
/* Move to next page */ /* Move to next page */
GenericXLogAbort(state); if (!building)
GenericXLogAbort(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
} }
else else
@@ -198,22 +216,31 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
Buffer newbuf; Buffer newbuf;
Page newpage; Page newpage;
HnswInsertAppendPage(index, &newbuf, &newpage, state, page); HnswInsertAppendPage(index, &newbuf, &newpage, state, page, building);
/* Commit */ /* Commit */
GenericXLogFinish(state); if (!building)
GenericXLogFinish(state);
/* Unlock previous buffer */ /* Unlock previous buffer */
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
/* Prepare new buffer */ /* Prepare new buffer */
state = GenericXLogStart(index);
buf = newbuf; buf = newbuf;
page = GenericXLogRegisterBuffer(state, buf, 0); if (building)
{
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Create new page for neighbors if needed */ /* Create new page for neighbors if needed */
if (PageGetFreeSpace(page) < combinedSize) if (PageGetFreeSpace(page) < combinedSize)
HnswInsertAppendPage(index, &nbuf, &npage, state, page); HnswInsertAppendPage(index, &nbuf, &npage, state, page, building);
else else
{ {
nbuf = buf; nbuf = buf;
@@ -267,7 +294,8 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
} }
/* Commit */ /* Commit */
GenericXLogFinish(state); if (!building)
GenericXLogFinish(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
if (nbuf != buf) if (nbuf != buf)
UnlockReleaseBuffer(nbuf); UnlockReleaseBuffer(nbuf);
@@ -301,7 +329,7 @@ ConnectionExists(HnswElement e, HnswNeighborTuple ntup, int startIdx, int lm)
* Update neighbors * Update neighbors
*/ */
void void
HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting) HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building)
{ {
for (int lc = e->level; lc >= 0; lc--) for (int lc = e->level; lc >= 0; lc--)
{ {
@@ -342,8 +370,16 @@ HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswE
/* Register page */ /* Register page */
buf = ReadBuffer(index, hc->element->neighborPage); buf = ReadBuffer(index, hc->element->neighborPage);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
state = GenericXLogStart(index); if (building)
page = GenericXLogRegisterBuffer(state, buf, 0); {
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Get tuple */ /* Get tuple */
itemid = PageGetItemId(page, offno); itemid = PageGetItemId(page, offno);
@@ -385,9 +421,10 @@ HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswE
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index)); elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Commit */ /* Commit */
GenericXLogFinish(state); if (!building)
GenericXLogFinish(state);
} }
else else if (!building)
GenericXLogAbort(state); GenericXLogAbort(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
@@ -399,7 +436,7 @@ HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswE
* Add a heap TID to an existing element * Add a heap TID to an existing element
*/ */
static bool static bool
HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup) HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup, bool building)
{ {
Buffer buf; Buffer buf;
Page page; Page page;
@@ -412,8 +449,16 @@ HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup)
/* Read page */ /* Read page */
buf = ReadBuffer(index, dup->blkno); buf = ReadBuffer(index, dup->blkno);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
state = GenericXLogStart(index); if (building)
page = GenericXLogRegisterBuffer(state, buf, 0); {
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Find space */ /* Find space */
itemid = PageGetItemId(page, dup->offno); itemid = PageGetItemId(page, dup->offno);
@@ -428,7 +473,8 @@ HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup)
/* Either being deleted or we lost our chance to another backend */ /* Either being deleted or we lost our chance to another backend */
if (i == 0 || i == HNSW_HEAPTIDS) if (i == 0 || i == HNSW_HEAPTIDS)
{ {
GenericXLogAbort(state); if (!building)
GenericXLogAbort(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
return false; return false;
} }
@@ -441,7 +487,8 @@ HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup)
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index)); elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Commit */ /* Commit */
GenericXLogFinish(state); if (!building)
GenericXLogFinish(state);
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
return true; return true;
@@ -451,37 +498,37 @@ HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup)
* Write changes to disk * Write changes to disk
*/ */
static void static void
WriteElement(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement dup, HnswElement entryPoint) WriteElement(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement dup, HnswElement entryPoint, bool building)
{ {
BlockNumber newInsertPage = InvalidBlockNumber; BlockNumber newInsertPage = InvalidBlockNumber;
/* Try to add to existing page */ /* Try to add to existing page */
if (dup != NULL) if (dup != NULL)
{ {
if (HnswAddDuplicate(index, element, dup)) if (HnswAddDuplicate(index, element, dup, building))
return; return;
} }
/* Write element and neighbor tuples */ /* Write element and neighbor tuples */
WriteNewElementPages(index, element, m, GetInsertPage(index), &newInsertPage); WriteNewElementPages(index, element, m, GetInsertPage(index), &newInsertPage, building);
/* Update insert page if needed */ /* Update insert page if needed */
if (BlockNumberIsValid(newInsertPage)) if (BlockNumberIsValid(newInsertPage))
HnswUpdateMetaPage(index, 0, NULL, newInsertPage, MAIN_FORKNUM); HnswUpdateMetaPage(index, 0, NULL, newInsertPage, MAIN_FORKNUM, building);
/* Update neighbors */ /* Update neighbors */
HnswUpdateNeighborPages(index, procinfo, collation, element, m, false); HnswUpdateNeighborPages(index, procinfo, collation, element, m, false, building);
/* Update metapage if needed */ /* Update metapage if needed */
if (entryPoint == NULL || element->level > entryPoint->level) if (entryPoint == NULL || element->level > entryPoint->level)
HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_GREATER, element, InvalidBlockNumber, MAIN_FORKNUM); HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_GREATER, element, InvalidBlockNumber, MAIN_FORKNUM, building);
} }
/* /*
* Insert a tuple into the index * Insert a tuple into the index
*/ */
bool bool
HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel) HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel, bool building)
{ {
Datum value; Datum value;
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
@@ -540,7 +587,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
dup = HnswFindDuplicate(element); dup = HnswFindDuplicate(element);
/* Write to disk */ /* Write to disk */
WriteElement(index, procinfo, collation, element, m, efConstruction, dup, entryPoint); WriteElement(index, procinfo, collation, element, m, efConstruction, dup, entryPoint, building);
/* Release lock */ /* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode); UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);
@@ -574,7 +621,7 @@ hnswinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
oldCtx = MemoryContextSwitchTo(insertCtx); oldCtx = MemoryContextSwitchTo(insertCtx);
/* Insert tuple */ /* Insert tuple */
HnswInsertTuple(index, values, isnull, heap_tid, heap); HnswInsertTuple(index, values, isnull, heap_tid, heap, false);
/* Delete memory context */ /* Delete memory context */
MemoryContextSwitchTo(oldCtx); MemoryContextSwitchTo(oldCtx);

View File

@@ -184,27 +184,6 @@ HnswInitPage(Buffer buf, Page page)
HnswPageGetOpaque(page)->page_id = HNSW_PAGE_ID; HnswPageGetOpaque(page)->page_id = HNSW_PAGE_ID;
} }
/*
* Init and register page
*/
void
HnswInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state)
{
*state = GenericXLogStart(index);
*page = GenericXLogRegisterBuffer(*state, *buf, GENERIC_XLOG_FULL_IMAGE);
HnswInitPage(*buf, *page);
}
/*
* Commit buffer
*/
void
HnswCommitBuffer(Buffer buf, GenericXLogState *state)
{
GenericXLogFinish(state);
UnlockReleaseBuffer(buf);
}
/* /*
* Allocate neighbors * Allocate neighbors
*/ */
@@ -379,7 +358,7 @@ HnswUpdateMetaPageInfo(Page page, int updateEntry, HnswElement entryPoint, Block
* Update the metapage * Update the metapage
*/ */
void void
HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum) HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building)
{ {
Buffer buf; Buffer buf;
Page page; Page page;
@@ -387,12 +366,22 @@ HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, Bloc
buf = ReadBufferExtended(index, forkNum, HNSW_METAPAGE_BLKNO, RBM_NORMAL, NULL); buf = ReadBufferExtended(index, forkNum, HNSW_METAPAGE_BLKNO, RBM_NORMAL, NULL);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
state = GenericXLogStart(index); if (building)
page = GenericXLogRegisterBuffer(state, buf, 0); {
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
HnswUpdateMetaPageInfo(page, updateEntry, entryPoint, insertPage); HnswUpdateMetaPageInfo(page, updateEntry, entryPoint, insertPage);
HnswCommitBuffer(buf, state); if (!building)
GenericXLogFinish(state);
UnlockReleaseBuffer(buf);
} }
/* /*

View File

@@ -230,7 +230,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
/* Update neighbors */ /* Update neighbors */
HnswUpdateNeighborPages(index, procinfo, collation, element, m, true); HnswUpdateNeighborPages(index, procinfo, collation, element, m, true, false);
} }
/* /*
@@ -286,7 +286,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
* point is outdated and empty, the entry point will be empty * point is outdated and empty, the entry point will be empty
* until an element is repaired. * until an element is repaired.
*/ */
HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_ALWAYS, highestPoint, InvalidBlockNumber, MAIN_FORKNUM); HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_ALWAYS, highestPoint, InvalidBlockNumber, MAIN_FORKNUM, false);
} }
else else
{ {
@@ -419,7 +419,7 @@ RepairGraph(HnswVacuumState * vacuumstate)
* was replaced and highest point was outdated. * was replaced and highest point was outdated.
*/ */
if (entryPoint == NULL || element->level > entryPoint->level) if (entryPoint == NULL || element->level > entryPoint->level)
HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_GREATER, element, InvalidBlockNumber, MAIN_FORKNUM); HnswUpdateMetaPage(index, HNSW_UPDATE_ENTRY_GREATER, element, InvalidBlockNumber, MAIN_FORKNUM, false);
/* Release lock */ /* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode); UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);
@@ -564,7 +564,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
} }
/* Update insert page last, after everything has been marked as deleted */ /* Update insert page last, after everything has been marked as deleted */
HnswUpdateMetaPage(index, 0, NULL, insertPage, MAIN_FORKNUM); HnswUpdateMetaPage(index, 0, NULL, insertPage, MAIN_FORKNUM, false);
} }
/* /*