Switched to TidStore for HNSW vacuuming for Postgres 17+ [skip ci]

Co-authored-by: Itai Spiegel <itai@mave.com>

CHANGELOG.md

@@ -1,7 +1,12 @@
+## 0.8.5 (unreleased)
+
+- Reduced memory usage for small tables for IVFFlat index builds
+
 ## 0.8.4 (2026-06-30)
 
 - Fixed `hnsw graph not repaired` error with HNSW vacuuming
 - Fixed possible error with inserts during HNSW vacuuming
+- Fixed memory exceeding `maintenance_work_mem` with IVFFlat index builds
 
 ## 0.8.3 (2026-06-17)
 

src/hnsw.c

@@ -290,7 +290,11 @@ hnswhandler(PG_FUNCTION_ARGS)
 		.amcanparallel = false,
 		.amcanbuildparallel = true,
 		.amcaninclude = false,
+#if PG_VERSION_NUM >= 170000
+		.amusemaintenanceworkmem = true,
+#else
 		.amusemaintenanceworkmem = false,
+#endif
 		.amsummarizing = false,
 		.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
 		.amkeytype = InvalidOid,
@@ -351,7 +355,11 @@ hnswhandler(PG_FUNCTION_ARGS)
 	amroutine->amcanbuildparallel = true;
 #endif
 	amroutine->amcaninclude = false;
-	amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
+#if PG_VERSION_NUM >= 170000
+	amroutine->amusemaintenanceworkmem = true;
+#else
+	amroutine->amusemaintenanceworkmem = false;
+#endif
 #if PG_VERSION_NUM >= 160000
 	amroutine->amsummarizing = false;
 #endif

src/hnsw.h

@@ -26,6 +26,12 @@
 typedef Pointer Item;
 #endif
 
+#if PG_VERSION_NUM >= 170000
+#define HnswTidStore TidStore
+#else
+#define HnswTidStore tidhash_hash
+#endif
+
 #define HNSW_MAX_DIM 2000
 #define HNSW_MAX_NNZ 1000
 
@@ -427,7 +433,7 @@ typedef struct HnswVacuumState
 	HnswSupport support;
 
 	/* Variables */
-	struct tidhash_hash *deleting;
+	struct HnswTidStore *deleting;
 	BufferAccessStrategy bas;
 	HnswNeighborTuple ntup;
 	HnswElementData highestPoint;

src/hnswvacuum.c

@@ -10,6 +10,12 @@
 #include "utils/memutils.h"
 #include "utils/rel.h"
 
+#if PG_VERSION_NUM >= 170000
+#include "access/tidstore.h"
+#include "miscadmin.h"
+#include "postmaster/autovacuum.h"
+#endif
+
 #if PG_VERSION_NUM >= 160000
 #include "varatt.h"
 #endif
@@ -22,9 +28,13 @@
  * Check if deletion list contains an element
  */
 static bool
-DeletingElement(tidhash_hash * deleting, ItemPointer indextid)
+DeletingElement(HnswTidStore * deleting, ItemPointer indextid)
 {
+#if PG_VERSION_NUM >= 170000
+	return TidStoreIsMember(deleting, indextid);
+#else
 	return tidhash_lookup(deleting, *indextid) != NULL;
+#endif
 }
 
 /*
@@ -60,6 +70,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 		OffsetNumber offno;
 		OffsetNumber maxoffno;
 		bool		updated = false;
+#if PG_VERSION_NUM >= 170000
+		OffsetNumber deletedoffs[MaxOffsetNumber];
+		int			ndeletedoffs = 0;
+#endif
 
 		vacuum_delay_point();
 
@@ -121,6 +135,9 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 			if (!ItemPointerIsValid(&etup->heaptids[0]))
 			{
+#if PG_VERSION_NUM >= 170000
+				deletedoffs[ndeletedoffs++] = offno;
+#else
 				ItemPointerData indextid;
 				bool		found;
 
@@ -129,6 +146,7 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 				tidhash_insert(vacuumstate->deleting, indextid, &found);
 				Assert(!found);
+#endif
 			}
 			else if (etup->level > highestLevel)
 			{
@@ -157,6 +175,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 			}
 		}
 
+#if PG_VERSION_NUM >= 170000
+		TidStoreSetBlockOffsets(vacuumstate->deleting, blkno, deletedoffs, ndeletedoffs);
+#endif
+
 		blkno = HnswPageGetOpaque(page)->nextblkno;
 
 		if (updated)
@@ -756,7 +778,11 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 	HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
 
 	/* Create hash table */
+#if PG_VERSION_NUM >= 170000
+	vacuumstate->deleting = TidStoreCreateLocal((AmAutoVacuumWorkerProcess() && autovacuum_work_mem != -1) ? autovacuum_work_mem : maintenance_work_mem, true);
+#else
 	vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
+#endif
 }
 
 /*
@@ -765,7 +791,11 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 static void
 FreeVacuumState(HnswVacuumState * vacuumstate)
 {
+#if PG_VERSION_NUM >= 170000
+	TidStoreDestroy(vacuumstate->deleting);
+#else
 	tidhash_destroy(vacuumstate->deleting);
+#endif
 	FreeAccessStrategy(vacuumstate->bas);
 	pfree(vacuumstate->ntup);
 	MemoryContextDelete(vacuumstate->tmpCtx);

src/ivfbuild.c

@@ -438,15 +438,22 @@ ComputeCenters(IvfflatBuildState * buildstate)
 
 	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
 
-	/* Target 50 samples per list, with at least 10000 samples */
-	/* The number of samples has a large effect on index build time */
-	numSamples = buildstate->lists * 50;
-	if (numSamples < 10000)
-		numSamples = 10000;
-
 	/* Skip samples for unlogged table */
 	if (buildstate->heap == NULL)
 		numSamples = 1;
+	else
+	{
+		int64		maxTuples = (int64) RelationGetNumberOfBlocks(buildstate->heap) * MaxHeapTuplesPerPage;
+
+		/* Target 50 samples per list, with at least 10000 samples */
+		/* The number of samples has a large effect on index build time */
+		numSamples = buildstate->lists * 50;
+		if (numSamples < 10000)
+			numSamples = 10000;
+
+		/* Save memory since will not have more than max tuples */
+		numSamples = Max(Min(numSamples, maxTuples), 1);
+	}
 
 	/* Sample rows */
 	buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize);

test/expected/hnsw_bit.out

@@ -49,3 +49,11 @@ CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
 CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
 ERROR:  column cannot have more than 64000 dimensions for hnsw index
 DROP TABLE t;
+-- dimensions
+CREATE TABLE t (val bit(64000));
+CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
+DROP TABLE t;
+CREATE TABLE t (val bit(64001));
+CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
+ERROR:  column cannot have more than 64000 dimensions for hnsw index
+DROP TABLE t;

test/expected/hnsw_halfvec.out

@@ -100,3 +100,11 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t
 (1 row)
 
 DROP TABLE t;
+-- dimensions
+CREATE TABLE t (val halfvec(4000));
+CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
+DROP TABLE t;
+CREATE TABLE t (val halfvec(4001));
+CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
+ERROR:  column cannot have more than 4000 dimensions for hnsw index
+DROP TABLE t;

test/expected/hnsw_vector.out

@@ -161,6 +161,7 @@ ERROR:  value 1001 out of bounds for option "ef_construction"
 DETAIL:  Valid values are between "4" and "1000".
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
 ERROR:  ef_construction must be greater than or equal to 2 * m
+DROP TABLE t;
 SHOW hnsw.ef_search;
  hnsw.ef_search 
 ----------------
@@ -198,4 +199,11 @@ SET hnsw.scan_mem_multiplier = 0;
 ERROR:  0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
 SET hnsw.scan_mem_multiplier = 1001;
 ERROR:  1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
+-- dimensions
+CREATE TABLE t (val vector(2000));
+CREATE INDEX ON t USING hnsw (val vector_l2_ops);
+DROP TABLE t;
+CREATE TABLE t (val vector(2001));
+CREATE INDEX ON t USING hnsw (val vector_l2_ops);
+ERROR:  column cannot have more than 2000 dimensions for hnsw index
 DROP TABLE t;

test/expected/ivfflat_bit.out

@@ -35,3 +35,32 @@ NOTICE:  ivfflat index created with little data
 DETAIL:  This will cause low recall.
 HINT:  Drop the index until the table has more data.
 DROP TABLE t;
+-- dimensions
+CREATE TABLE t (val bit(64000));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+CREATE TABLE t (val bit(64001));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+ERROR:  column cannot have more than 64000 dimensions for ivfflat index
+DROP TABLE t;
+-- memory
+SET maintenance_work_mem = '1MB';
+CREATE TABLE t (val bit(64000));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+RESET maintenance_work_mem;
+SET maintenance_work_mem = '29MB';
+CREATE TABLE t (val bit(64000));
+INSERT INTO t (val) VALUES (B'0'::bit(64000));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+RESET maintenance_work_mem;

test/expected/ivfflat_halfvec.out

@@ -82,3 +82,32 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t
 (1 row)
 
 DROP TABLE t;
+-- dimensions
+CREATE TABLE t (val halfvec(4000));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+CREATE TABLE t (val halfvec(4001));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+ERROR:  column cannot have more than 4000 dimensions for ivfflat index
+DROP TABLE t;
+-- memory
+SET maintenance_work_mem = '1MB';
+CREATE TABLE t (val halfvec(4000));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+RESET maintenance_work_mem;
+SET maintenance_work_mem = '6MB';
+CREATE TABLE t (val halfvec(4000));
+INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+RESET maintenance_work_mem;

test/expected/ivfflat_vector.out

@@ -143,6 +143,7 @@ DETAIL:  Valid values are between "1" and "32768".
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
 ERROR:  value 32769 out of bounds for option "lists"
 DETAIL:  Valid values are between "1" and "32768".
+DROP TABLE t;
 SHOW ivfflat.probes;
  ivfflat.probes 
 ----------------
@@ -172,4 +173,32 @@ SET ivfflat.max_probes = 0;
 ERROR:  0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
 SET ivfflat.max_probes = 32769;
 ERROR:  32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
+-- dimensions
+CREATE TABLE t (val vector(2000));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
 DROP TABLE t;
+CREATE TABLE t (val vector(2001));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+ERROR:  column cannot have more than 2000 dimensions for ivfflat index
+DROP TABLE t;
+-- memory
+SET maintenance_work_mem = '1MB';
+CREATE TABLE t (val vector(2000));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+RESET maintenance_work_mem;
+SET maintenance_work_mem = '5MB';
+CREATE TABLE t (val vector(2000));
+INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+NOTICE:  ivfflat index created with little data
+DETAIL:  This will cause low recall.
+HINT:  Drop the index until the table has more data.
+DROP TABLE t;
+RESET maintenance_work_mem;

test/sql/hnsw_bit.sql

@@ -33,3 +33,13 @@ CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
 CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
 CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
 DROP TABLE t;
+
+-- dimensions
+
+CREATE TABLE t (val bit(64000));
+CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
+DROP TABLE t;
+
+CREATE TABLE t (val bit(64001));
+CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
+DROP TABLE t;

test/sql/hnsw_halfvec.sql

@@ -56,3 +56,13 @@ SELECT * FROM t ORDER BY val <+> '[3,3,3]';
 SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
 
 DROP TABLE t;
+
+-- dimensions
+
+CREATE TABLE t (val halfvec(4000));
+CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
+DROP TABLE t;
+
+CREATE TABLE t (val halfvec(4001));
+CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
+DROP TABLE t;

test/sql/hnsw_vector.sql

@@ -95,23 +95,28 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
+DROP TABLE t;
 
 SHOW hnsw.ef_search;
-
 SET hnsw.ef_search = 0;
 SET hnsw.ef_search = 1001;
 
 SHOW hnsw.iterative_scan;
-
 SET hnsw.iterative_scan = on;
 
 SHOW hnsw.max_scan_tuples;
-
 SET hnsw.max_scan_tuples = 0;
 
 SHOW hnsw.scan_mem_multiplier;
-
 SET hnsw.scan_mem_multiplier = 0;
 SET hnsw.scan_mem_multiplier = 1001;
 
+-- dimensions
+
+CREATE TABLE t (val vector(2000));
+CREATE INDEX ON t USING hnsw (val vector_l2_ops);
+DROP TABLE t;
+
+CREATE TABLE t (val vector(2001));
+CREATE INDEX ON t USING hnsw (val vector_l2_ops);
 DROP TABLE t;

test/sql/ivfflat_bit.sql

@@ -21,3 +21,28 @@ CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1)
 CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
 CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
 DROP TABLE t;
+
+-- dimensions
+
+CREATE TABLE t (val bit(64000));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+DROP TABLE t;
+
+CREATE TABLE t (val bit(64001));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+DROP TABLE t;
+
+-- memory
+
+SET maintenance_work_mem = '1MB';
+CREATE TABLE t (val bit(64000));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+DROP TABLE t;
+RESET maintenance_work_mem;
+
+SET maintenance_work_mem = '29MB';
+CREATE TABLE t (val bit(64000));
+INSERT INTO t (val) VALUES (B'0'::bit(64000));
+CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
+DROP TABLE t;
+RESET maintenance_work_mem;

test/sql/ivfflat_halfvec.sql

@@ -43,3 +43,28 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
 SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
 
 DROP TABLE t;
+
+-- dimensions
+
+CREATE TABLE t (val halfvec(4000));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+DROP TABLE t;
+
+CREATE TABLE t (val halfvec(4001));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+DROP TABLE t;
+
+-- memory
+
+SET maintenance_work_mem = '1MB';
+CREATE TABLE t (val halfvec(4000));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+DROP TABLE t;
+RESET maintenance_work_mem;
+
+SET maintenance_work_mem = '6MB';
+CREATE TABLE t (val halfvec(4000));
+INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
+CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
+DROP TABLE t;
+RESET maintenance_work_mem;

test/sql/ivfflat_vector.sql

@@ -81,19 +81,40 @@ DROP TABLE t;
 CREATE TABLE t (val vector(3));
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
+DROP TABLE t;
 
 SHOW ivfflat.probes;
-
 SET ivfflat.probes = 0;
 SET ivfflat.probes = 32769;
 
 SHOW ivfflat.iterative_scan;
-
 SET ivfflat.iterative_scan = on;
 
 SHOW ivfflat.max_probes;
-
 SET ivfflat.max_probes = 0;
 SET ivfflat.max_probes = 32769;
 
+-- dimensions
+
+CREATE TABLE t (val vector(2000));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
 DROP TABLE t;
+
+CREATE TABLE t (val vector(2001));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+DROP TABLE t;
+
+-- memory
+
+SET maintenance_work_mem = '1MB';
+CREATE TABLE t (val vector(2000));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+DROP TABLE t;
+RESET maintenance_work_mem;
+
+SET maintenance_work_mem = '5MB';
+CREATE TABLE t (val vector(2000));
+INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
+CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
+DROP TABLE t;
+RESET maintenance_work_mem;