diff --git a/CHANGELOG.md b/CHANGELOG.md
index d4aadb7..284c4c0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.2.4 (unreleased)
+
+- Fixed issue with index not reusing space
+
 ## 0.2.3 (2022-01-30)
 
 - Added indexing progress for Postgres 12+
diff --git a/src/ivfbuild.c b/src/ivfbuild.c
index c9264e8..ecc65c4 100644
--- a/src/ivfbuild.c
+++ b/src/ivfbuild.c
@@ -288,7 +288,7 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 		IvfflatCommitBuffer(buf, state);
 
 		/* Set the start and insert pages */
-		IvfflatUpdateList(index, state, buildstate->listInfo[i], insertPage, startPage, forkNum);
+		IvfflatUpdateList(index, state, buildstate->listInfo[i], insertPage, InvalidBlockNumber, startPage, forkNum);
 	}
 }
 
diff --git a/src/ivfflat.h b/src/ivfflat.h
index 79cea6c..a6a9a5b 100644
--- a/src/ivfflat.h
+++ b/src/ivfflat.h
@@ -177,7 +177,7 @@ void		IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
 FmgrInfo   *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);
 bool		IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
 int			IvfflatGetLists(Relation index);
-void		IvfflatUpdateList(Relation index, GenericXLogState *state, ListInfo listInfo, BlockNumber insertPage, BlockNumber startPage, ForkNumber forkNum);
+void		IvfflatUpdateList(Relation index, GenericXLogState *state, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
 void		IvfflatCommitBuffer(Buffer buf, GenericXLogState *state);
 void		IvfflatAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state, ForkNumber forkNum);
 Buffer		IvfflatNewBuffer(Relation index, ForkNumber forkNum);
diff --git a/src/ivfinsert.c b/src/ivfinsert.c
index 9e63662..449d141 100644
--- a/src/ivfinsert.c
+++ b/src/ivfinsert.c
@@ -77,11 +77,12 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
 	Size		itemsz;
 	BlockNumber insertPage = InvalidBlockNumber;
 	ListInfo	listInfo;
-	bool		newPage = false;
+	BlockNumber originalInsertPage;
 
 	/* Find the insert page - sets the page and list info */
 	FindInsertPage(rel, values, &insertPage, &listInfo);
 	Assert(BlockNumberIsValid(insertPage));
+	originalInsertPage = insertPage;
 
 	itemsz = MAXALIGN(IndexTupleSize(itup));
 	Assert(itemsz <= BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(IvfflatPageOpaqueData)));
@@ -107,7 +108,6 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
 			IvfflatAppendPage(rel, &buf, &page, &state, MAIN_FORKNUM);
 
 			insertPage = BufferGetBlockNumber(buf);
-			newPage = true;
 		}
 	}
 
@@ -118,8 +118,8 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
 	IvfflatCommitBuffer(buf, state);
 
 	/* Update the insert page */
-	if (newPage)
-		IvfflatUpdateList(rel, state, listInfo, insertPage, InvalidBlockNumber, MAIN_FORKNUM);
+	if (insertPage != originalInsertPage)
+		IvfflatUpdateList(rel, state, listInfo, insertPage, originalInsertPage, InvalidBlockNumber, MAIN_FORKNUM);
 }
 
 /*
diff --git a/src/ivfutils.c b/src/ivfutils.c
index f3a2cf9..33b1df7 100644
--- a/src/ivfutils.c
+++ b/src/ivfutils.c
@@ -153,11 +153,13 @@ IvfflatAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **st
  */
 void
 IvfflatUpdateList(Relation index, GenericXLogState *state, ListInfo listInfo,
-				  BlockNumber insertPage, BlockNumber startPage, ForkNumber forkNum)
+				  BlockNumber insertPage, BlockNumber originalInsertPage,
+				  BlockNumber startPage, ForkNumber forkNum)
 {
 	Buffer		buf;
 	Page		page;
 	IvfflatList list;
+	bool		changed = false;
 
 	buf = ReadBufferExtended(index, forkNum, listInfo.blkno, RBM_NORMAL, NULL);
 	LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
@@ -165,12 +167,29 @@ IvfflatUpdateList(Relation index, GenericXLogState *state, ListInfo listInfo,
 	page = GenericXLogRegisterBuffer(state, buf, 0);
 	list = (IvfflatList) PageGetItem(page, PageGetItemId(page, listInfo.offno));
 
-	if (BlockNumberIsValid(insertPage))
-		list->insertPage = insertPage;
+	if (BlockNumberIsValid(insertPage) && insertPage != list->insertPage)
+	{
+		/* Skip update if insert page is lower than original insert page  */
+		/* This is needed to prevent insert from overwriting vacuum */
+		if (!BlockNumberIsValid(originalInsertPage) || insertPage >= originalInsertPage)
+		{
+			list->insertPage = insertPage;
+			changed = true;
+		}
+	}
 
-	if (BlockNumberIsValid(startPage))
+	if (BlockNumberIsValid(startPage) && startPage != list->startPage)
+	{
 		list->startPage = startPage;
+		changed = true;
+	}
 
-	/* Could only commit if changed, but extra complexity isn't needed */
-	IvfflatCommitBuffer(buf, state);
+	/* Only commit if changed */
+	if (changed)
+		IvfflatCommitBuffer(buf, state);
+	else
+	{
+		GenericXLogAbort(state);
+		UnlockReleaseBuffer(buf);
+	}
 }
diff --git a/src/ivfvacuum.c b/src/ivfvacuum.c
index 7c27bdf..4441c0d 100644
--- a/src/ivfvacuum.c
+++ b/src/ivfvacuum.c
@@ -21,7 +21,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 	ItemPointer htup;
 	OffsetNumber deletable[MaxOffsetNumber];
 	int			ndeletable;
-	OffsetNumber startPages[MaxOffsetNumber];
+	BlockNumber startPages[MaxOffsetNumber];
 	BlockNumber nextblkno = IVFFLAT_HEAD_BLKNO;
 	BlockNumber searchPage;
 	BlockNumber insertPage;
@@ -98,6 +98,11 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 						stats->num_index_tuples++;
 				}
 
+				/* Set to first free page */
+				/* Must be set before searchPage is updated */
+				if (!BlockNumberIsValid(insertPage) && ndeletable > 0)
+					insertPage = searchPage;
+
 				searchPage = IvfflatPageGetOpaque(page)->nextblkno;
 
 				if (ndeletable > 0)
@@ -106,10 +111,6 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 					PageIndexMultiDelete(page, deletable, ndeletable);
 					MarkBufferDirty(buf);
 					GenericXLogFinish(state);
-
-					/* Set to first free page */
-					if (!BlockNumberIsValid(insertPage))
-						insertPage = searchPage;
 				}
 				else
 					GenericXLogAbort(state);
@@ -123,10 +124,10 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 			 * We don't add or delete items from lists pages, so offset won't
 			 * change.
 			 */
-			if (!BlockNumberIsValid(insertPage))
+			if (BlockNumberIsValid(insertPage))
 			{
 				listInfo.offno = coffno;
-				IvfflatUpdateList(index, state, listInfo, insertPage, InvalidBlockNumber, MAIN_FORKNUM);
+				IvfflatUpdateList(index, state, listInfo, insertPage, InvalidBlockNumber, InvalidBlockNumber, MAIN_FORKNUM);
 			}
 		}
 	}