From 5db9581e740656cdda684ccda0255279e2bd4384 Mon Sep 17 00:00:00 2001
From: Andrew Kane <andrew@ankane.org>
Date: Fri, 20 Sep 2024 15:10:54 -0700
Subject: [PATCH] Added versioning to tuples

---
 src/hnsw.h       |  7 ++++---
 src/hnswinsert.c | 10 ++++++++--
 src/hnswutils.c  | 21 ++++++++++++++++++---
 src/hnswvacuum.c |  5 +++++
 4 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/src/hnsw.h b/src/hnsw.h
index 2f45039..d201b8c 100644
--- a/src/hnsw.h
+++ b/src/hnsw.h
@@ -129,6 +129,7 @@ struct HnswElementData
 	uint8		heaptidsLength;
 	uint8		level;
 	uint8		deleted;
+	uint8		version;
 	uint32		hash;
 	HnswNeighborsPtr neighbors;
 	BlockNumber blkno;
@@ -305,10 +306,10 @@ typedef struct HnswElementTupleData
 	uint8		type;
 	uint8		level;
 	uint8		deleted;
-	uint8		unused;
+	uint8		version;
 	ItemPointerData heaptids[HNSW_HEAPTIDS];
 	ItemPointerData neighbortid;
-	uint16		unused2;
+	uint16		unused;
 	Vector		data;
 }			HnswElementTupleData;
 
@@ -317,7 +318,7 @@ typedef HnswElementTupleData * HnswElementTuple;
 typedef struct HnswNeighborTupleData
 {
 	uint8		type;
-	uint8		unused;
+	uint8		version;
 	uint16		count;
 	ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
 }			HnswNeighborTupleData;
diff --git a/src/hnswinsert.c b/src/hnswinsert.c
index 2dce16f..fdc18c0 100644
--- a/src/hnswinsert.c
+++ b/src/hnswinsert.c
@@ -36,7 +36,7 @@ GetInsertPage(Relation index)
  * Check for a free offset
  */
 static bool
-HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
+HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage, uint8 *tupleVersion)
 {
 	OffsetNumber offno;
 	OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
@@ -98,6 +98,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
 			{
 				*freeOffno = offno;
 				*freeNeighborOffno = neighborOffno;
+				*tupleVersion = etup->version;
 				return true;
 			}
 			else if (*nbuf != buf)
@@ -153,6 +154,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 	OffsetNumber freeOffno = InvalidOffsetNumber;
 	OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
 	BlockNumber newInsertPage = InvalidBlockNumber;
+	uint8		tupleVersion;
 	char	   *base = NULL;
 
 	/* Calculate sizes */
@@ -202,7 +204,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 		}
 
 		/* Next, try space from a deleted element */
-		if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
+		if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage, &tupleVersion))
 		{
 			if (nbuf != buf)
 			{
@@ -212,6 +214,10 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 					npage = GenericXLogRegisterBuffer(state, nbuf, 0);
 			}
 
+			/* Set tuple version */
+			etup->version = tupleVersion;
+			ntup->version = tupleVersion;
+
 			break;
 		}
 
diff --git a/src/hnswutils.c b/src/hnswutils.c
index f69c057..2f5a39e 100644
--- a/src/hnswutils.c
+++ b/src/hnswutils.c
@@ -253,6 +253,8 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
 
 	element->level = level;
 	element->deleted = 0;
+	/* Start at one to make it easier to find issues */
+	element->version = 1;
 
 	HnswInitNeighbors(base, element, m, allocator);
 
@@ -405,6 +407,7 @@ HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element)
 	etup->type = HNSW_ELEMENT_TUPLE_TYPE;
 	etup->level = element->level;
 	etup->deleted = 0;
+	etup->version = element->version;
 	for (int i = 0; i < HNSW_HEAPTIDS; i++)
 	{
 		if (i < element->heaptidsLength)
@@ -447,6 +450,7 @@ HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m)
 	}
 
 	ntup->count = idx;
+	ntup->version = e->version;
 }
 
 /*
@@ -520,6 +524,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
 {
 	element->level = etup->level;
 	element->deleted = etup->deleted;
+	element->version = etup->version;
 	element->neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
 	element->neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
 	element->heaptidsLength = 0;
@@ -766,20 +771,30 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
 	int			start;
 	ItemPointerData indextids[HNSW_MAX_M * 2];
 
+	*unvisitedLength = 0;
+
 	buf = ReadBuffer(index, element->neighborPage);
 	LockBuffer(buf, BUFFER_LOCK_SHARE);
 	page = BufferGetPage(buf);
 
 	ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
-	start = (element->level - lc) * m;
+
+	/*
+	 * Ensure the neighbor tuple has not been deleted or replaced between
+	 * index scan iterations
+	 */
+	if (ntup->version != element->version)
+	{
+		UnlockReleaseBuffer(buf);
+		return;
+	}
 
 	/* Copy to minimize lock time */
+	start = (element->level - lc) * m;
 	memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
 
 	UnlockReleaseBuffer(buf);
 
-	*unvisitedLength = 0;
-
 	for (int i = 0; i < lm; i++)
 	{
 		ItemPointer indextid = &indextids[i];
diff --git a/src/hnswvacuum.c b/src/hnswvacuum.c
index 67cc645..b3bb718 100644
--- a/src/hnswvacuum.c
+++ b/src/hnswvacuum.c
@@ -527,6 +527,11 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 			for (int i = 0; i < ntup->count; i++)
 				ItemPointerSetInvalid(&ntup->indextids[i]);
 
+			/* Increment version */
+			/* This is used to avoid incorrect reads for iterative scans */
+			etup->version++;
+			ntup->version = etup->version;
+
 			/*
 			 * We modified the tuples in place, no need to call
 			 * PageIndexTupleOverwrite