diff --git a/src/hnsw.h b/src/hnsw.h
index 56f2ccf..a2d6562 100644
--- a/src/hnsw.h
+++ b/src/hnsw.h
@@ -49,7 +49,7 @@
 #define PROGRESS_HNSW_PHASE_LOAD		2
 
 #define HNSW_ELEMENT_TUPLE_SIZE(_dim)	MAXALIGN(offsetof(HnswElementTupleData, vec) + VECTOR_SIZE(_dim))
-#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m)	MAXALIGN(offsetof(HnswNeighborTupleData, neighbors) + ((level) + 2) * (m) * sizeof(HnswNeighborTupleItem))
+#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m)	MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
 
 #define HnswPageGetOpaque(page)	((HnswPageOpaque) PageGetSpecialPointer(page))
 #define HnswPageGetMeta(page)	((HnswMetaPageData *) PageGetContents(page))
@@ -197,19 +197,12 @@ typedef struct HnswElementTupleData
 
 typedef HnswElementTupleData * HnswElementTuple;
 
-typedef struct HnswNeighborTupleItem
-{
-	ItemPointerData indextid;
-	uint16		unused;
-	float		distance;		/* improves performance of inserts */
-}			HnswNeighborTupleItem;
-
 typedef struct HnswNeighborTupleData
 {
 	uint8		type;
 	uint8		unused;
 	uint16		count;
-	HnswNeighborTupleItem neighbors[FLEXIBLE_ARRAY_MEMBER];
+	ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
 }			HnswNeighborTupleData;
 
 typedef HnswNeighborTupleData * HnswNeighborTuple;
@@ -296,6 +289,6 @@ bool		hnswgettuple(IndexScanDesc scan, ScanDirection dir);
 void		hnswendscan(IndexScanDesc scan);
 
 /* Ensure fits in uint8 */
-#define HnswGetMaxLevel(m) Min(((BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - offsetof(HnswNeighborTupleData, neighbors) - sizeof(ItemIdData)) / (sizeof(HnswNeighborTupleItem)) / m) - 2, 255)
+#define HnswGetMaxLevel(m) Min(((BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - offsetof(HnswNeighborTupleData, indextids) - sizeof(ItemIdData)) / (sizeof(ItemPointerData)) / m) - 2, 255)
 
 #endif
diff --git a/src/hnswinsert.c b/src/hnswinsert.c
index c2a17ff..5dc4fb3 100644
--- a/src/hnswinsert.c
+++ b/src/hnswinsert.c
@@ -317,11 +317,10 @@ UpdateNeighborPages(Relation index, HnswElement e, int m, List *updates)
 		/* Make robust to issues */
 		if (idx < ntup->count)
 		{
-			HnswNeighborTupleItem *neighbor = &ntup->neighbors[idx];
+			ItemPointer indextid = &ntup->indextids[idx];
 
 			/* Update neighbor */
-			ItemPointerSet(&neighbor->indextid, e->blkno, e->offno);
-			neighbor->distance = update->hc.distance;
+			ItemPointerSet(indextid, e->blkno, e->offno);
 
 			/* Overwrite tuple */
 			if (!PageIndexTupleOverwrite(page, offno, (Item) ntup, ntupSize))
diff --git a/src/hnswutils.c b/src/hnswutils.c
index 1d3409e..3092144 100644
--- a/src/hnswutils.c
+++ b/src/hnswutils.c
@@ -309,20 +309,16 @@ HnswSetNeighborTuple(HnswNeighborTuple ntup, HnswElement e, int m)
 
 		for (int i = 0; i < lm; i++)
 		{
-			HnswNeighborTupleItem *neighbor = &ntup->neighbors[idx++];
+			ItemPointer indextid = &ntup->indextids[idx++];
 
 			if (i < neighbors->length)
 			{
 				HnswCandidate *hc = &neighbors->items[i];
 
-				ItemPointerSet(&neighbor->indextid, hc->element->blkno, hc->element->offno);
-				neighbor->distance = hc->distance;
+				ItemPointerSet(indextid, hc->element->blkno, hc->element->offno);
 			}
 			else
-			{
-				ItemPointerSetInvalid(&neighbor->indextid);
-				neighbor->distance = NAN;
-			}
+				ItemPointerSetInvalid(indextid);
 		}
 	}
 
@@ -352,15 +348,15 @@ LoadNeighborsFromPage(HnswElement element, Relation index, Page page)
 		HnswElement e;
 		int			level;
 		HnswCandidate *hc;
-		HnswNeighborTupleItem *neighbor;
+		ItemPointer indextid;
 		HnswNeighborArray *neighbors;
 
-		neighbor = &ntup->neighbors[i];
+		indextid = &ntup->indextids[i];
 
-		if (!ItemPointerIsValid(&neighbor->indextid))
+		if (!ItemPointerIsValid(indextid))
 			continue;
 
-		e = InitElementFromBlock(ItemPointerGetBlockNumber(&neighbor->indextid), ItemPointerGetOffsetNumber(&neighbor->indextid));
+		e = InitElementFromBlock(ItemPointerGetBlockNumber(indextid), ItemPointerGetOffsetNumber(indextid));
 
 		/* Calculate level based on offset */
 		level = element->level - i / m;
@@ -370,7 +366,6 @@ LoadNeighborsFromPage(HnswElement element, Relation index, Page page)
 		neighbors = &element->neighbors[level];
 		hc = &neighbors->items[neighbors->length++];
 		hc->element = e;
-		hc->distance = neighbor->distance;
 	}
 }
 
@@ -850,31 +845,35 @@ UpdateConnections(HnswElement element, List *neighbors, int m, int lc, List **up
 			HnswCandidate *pruned = NULL;
 			List	   *c = NIL;
 
+			/* Load elements on insert */
+			if (index != NULL)
+			{
+				Datum		q = PointerGetDatum(hc->element->vec);
+
+				for (int i = 0; i < currentNeighbors->length; i++)
+				{
+					HnswCandidate *hc3 = &currentNeighbors->items[i];
+
+					if (hc3->element->vec == NULL)
+						HnswLoadElement(hc3->element, &hc3->distance, &q, index, procinfo, collation, true);
+					else
+						hc3->distance = GetCandidateDistance(hc3, q, procinfo, collation);
+
+					/* Prune deleted element */
+					if (hc3->element->deleted)
+					{
+						pruned = &currentNeighbors->items[i];
+						break;
+					}
+				}
+			}
+
 			/* Add and sort candidates */
 			for (int i = 0; i < currentNeighbors->length; i++)
 				c = lappend(c, &currentNeighbors->items[i]);
 			c = lappend(c, &hc2);
 			list_sort(c, CompareCandidateDistances);
 
-			/* Load elements on insert */
-			if (index != NULL)
-			{
-				for (int i = 0; i < currentNeighbors->length; i++)
-				{
-					if (currentNeighbors->items[i].element->vec == NULL)
-					{
-						HnswLoadElement(currentNeighbors->items[i].element, NULL, NULL, index, procinfo, collation, true);
-
-						/* Prune deleted element */
-						if (currentNeighbors->items[i].element->deleted)
-						{
-							pruned = &currentNeighbors->items[i];
-							break;
-						}
-					}
-				}
-			}
-
 			if (pruned == NULL)
 			{
 				SelectNeighbors(c, m, lc, procinfo, collation, &pruned);
diff --git a/src/hnswvacuum.c b/src/hnswvacuum.c
index b37c362..c1c289d 100644
--- a/src/hnswvacuum.c
+++ b/src/hnswvacuum.c
@@ -156,13 +156,13 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
 	/* Check neighbors */
 	for (int i = 0; i < ntup->count; i++)
 	{
-		HnswNeighborTupleItem *neighbor = &ntup->neighbors[i];
+		ItemPointer indextid = &ntup->indextids[i];
 
-		if (!ItemPointerIsValid(&neighbor->indextid))
+		if (!ItemPointerIsValid(indextid))
 			continue;
 
 		/* Check if in deleted list */
-		if (DeletedContains(vacuumstate->deleted, &neighbor->indextid))
+		if (DeletedContains(vacuumstate->deleted, indextid))
 		{
 			needsUpdated = true;
 			break;
@@ -453,10 +453,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 
 			/* Overwrite neighbors */
 			for (int i = 0; i < ntup->count; i++)
-			{
-				ItemPointerSetInvalid(&ntup->neighbors[i].indextid);
-				ntup->neighbors[i].distance = NAN;
-			}
+				ItemPointerSetInvalid(&ntup->indextids[i]);
 
 			/* Overwrite element tuple */
 			if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))