Switched to TidStore for HNSW vacuuming for Postgres 17+ [skip ci]

.github/workflows/build.yml

@@ -8,12 +8,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - postgres: 20
-            os: ubuntu-26.04
           - postgres: 19
-            os: ubuntu-26.04
+            os: ubuntu-24.04
           - postgres: 18
-            os: ubuntu-26.04-arm
+            os: ubuntu-24.04
           - postgres: 17
             os: ubuntu-24.04
           - postgres: 16
@@ -25,7 +23,7 @@ jobs:
           - postgres: 13
             os: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: ${{ matrix.postgres }}
@@ -55,7 +53,7 @@ jobs:
           - postgres: 14
             os: macos-15-intel
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: ${{ matrix.postgres }}
@@ -94,7 +92,7 @@ jobs:
           - postgres: 14
             os: windows-2022
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: ${{ matrix.postgres }}
@@ -135,7 +133,7 @@ jobs:
     if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
       - uses: ankane/setup-postgres-valgrind@v1
         with:
           postgres-version: 18

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.8.5 (2026-07-08)
+## 0.8.5 (unreleased)
 
 - Reduced memory usage for small tables for IVFFlat index builds
 

Dockerfile

@@ -5,7 +5,7 @@ ARG DEBIAN_CODENAME=bookworm
 FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
 ARG PG_MAJOR
 
-ADD https://github.com/pgvector/pgvector.git#v0.8.5 /tmp/pgvector
+ADD https://github.com/pgvector/pgvector.git#v0.8.4 /tmp/pgvector
 
 RUN apt-get update && \
 		apt-mark hold locales && \

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.8.5",
+	"version": "0.8.4",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.8.5",
+			"version": "0.8.4",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.8.5
+EXTVERSION = 0.8.4
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*--*.sql)

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.8.5
+EXTVERSION = 0.8.4
 
 DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
 OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

README.md

@@ -23,7 +23,7 @@ Compile and install the extension (supports Postgres 13+)
 
 ```sh
 cd /tmp
-git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
+git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -40,7 +40,7 @@ Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/buil
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\18"
 cd %TEMP%
-git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
+git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -465,16 +465,6 @@ If filtering by many different values, consider [partitioning](https://www.postg
 CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
 ```
 
-## Multitenancy
-
-For applications with multiple tenants, sharing an approximate index between tenants means vectors from one tenant can affect recall (and speed) for other tenants.
-
-For tenant isolation, use [list partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) or separate tables.
-
-```sql
-CREATE TABLE items (customer_id int, embedding vector(3)) PARTITION BY LIST(customer_id);
-```
-
 ## Iterative Index Scans
 
 With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
@@ -1161,23 +1151,23 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 
 Supported tags are:
 
-- `pg18-trixie`, `0.8.5-pg18-trixie`
-- `pg18-bookworm`, `0.8.5-pg18-bookworm`, `pg18`, `0.8.5-pg18`
-- `pg17-trixie`, `0.8.5-pg17-trixie`
-- `pg17-bookworm`, `0.8.5-pg17-bookworm`, `pg17`, `0.8.5-pg17`
-- `pg16-trixie`, `0.8.5-pg16-trixie`
-- `pg16-bookworm`, `0.8.5-pg16-bookworm`, `pg16`, `0.8.5-pg16`
-- `pg15-trixie`, `0.8.5-pg15-trixie`
-- `pg15-bookworm`, `0.8.5-pg15-bookworm`, `pg15`, `0.8.5-pg15`
-- `pg14-trixie`, `0.8.5-pg14-trixie`
-- `pg14-bookworm`, `0.8.5-pg14-bookworm`, `pg14`, `0.8.5-pg14`
-- `pg13-trixie`, `0.8.5-pg13-trixie`
-- `pg13-bookworm`, `0.8.5-pg13-bookworm`, `pg13`, `0.8.5-pg13`
+- `pg18-trixie`, `0.8.4-pg18-trixie`
+- `pg18-bookworm`, `0.8.4-pg18-bookworm`, `pg18`, `0.8.4-pg18`
+- `pg17-trixie`, `0.8.4-pg17-trixie`
+- `pg17-bookworm`, `0.8.4-pg17-bookworm`, `pg17`, `0.8.4-pg17`
+- `pg16-trixie`, `0.8.4-pg16-trixie`
+- `pg16-bookworm`, `0.8.4-pg16-bookworm`, `pg16`, `0.8.4-pg16`
+- `pg15-trixie`, `0.8.4-pg15-trixie`
+- `pg15-bookworm`, `0.8.4-pg15-bookworm`, `pg15`, `0.8.4-pg15`
+- `pg14-trixie`, `0.8.4-pg14-trixie`
+- `pg14-bookworm`, `0.8.4-pg14-bookworm`, `pg14`, `0.8.4-pg14`
+- `pg13-trixie`, `0.8.4-pg13-trixie`
+- `pg13-bookworm`, `0.8.4-pg13-bookworm`, `pg13`, `0.8.4-pg13`
 
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
+git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
 ```

sql/vector--0.8.4--0.8.5.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.5'" to load this file. \quit

src/hnsw.c

@@ -290,7 +290,11 @@ hnswhandler(PG_FUNCTION_ARGS)
 		.amcanparallel = false,
 		.amcanbuildparallel = true,
 		.amcaninclude = false,
+#if PG_VERSION_NUM >= 170000
+		.amusemaintenanceworkmem = true,
+#else
 		.amusemaintenanceworkmem = false,
+#endif
 		.amsummarizing = false,
 		.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
 		.amkeytype = InvalidOid,
@@ -351,7 +355,11 @@ hnswhandler(PG_FUNCTION_ARGS)
 	amroutine->amcanbuildparallel = true;
 #endif
 	amroutine->amcaninclude = false;
-	amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
+#if PG_VERSION_NUM >= 170000
+	amroutine->amusemaintenanceworkmem = true;
+#else
+	amroutine->amusemaintenanceworkmem = false;
+#endif
 #if PG_VERSION_NUM >= 160000
 	amroutine->amsummarizing = false;
 #endif

src/hnsw.h

@@ -26,6 +26,12 @@
 typedef Pointer Item;
 #endif
 
+#if PG_VERSION_NUM >= 170000
+#define HnswTidStore TidStore
+#else
+#define HnswTidStore tidhash_hash
+#endif
+
 #define HNSW_MAX_DIM 2000
 #define HNSW_MAX_NNZ 1000
 
@@ -427,7 +433,7 @@ typedef struct HnswVacuumState
 	HnswSupport support;
 
 	/* Variables */
-	struct tidhash_hash *deleting;
+	struct HnswTidStore *deleting;
 	BufferAccessStrategy bas;
 	HnswNeighborTuple ntup;
 	HnswElementData highestPoint;

src/hnswvacuum.c

@@ -10,6 +10,12 @@
 #include "utils/memutils.h"
 #include "utils/rel.h"
 
+#if PG_VERSION_NUM >= 170000
+#include "access/tidstore.h"
+#include "miscadmin.h"
+#include "postmaster/autovacuum.h"
+#endif
+
 #if PG_VERSION_NUM >= 160000
 #include "varatt.h"
 #endif
@@ -22,9 +28,13 @@
  * Check if deletion list contains an element
  */
 static bool
-DeletingElement(tidhash_hash * deleting, ItemPointer indextid)
+DeletingElement(HnswTidStore * deleting, ItemPointer indextid)
 {
+#if PG_VERSION_NUM >= 170000
+	return TidStoreIsMember(deleting, indextid);
+#else
 	return tidhash_lookup(deleting, *indextid) != NULL;
+#endif
 }
 
 /*
@@ -60,6 +70,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 		OffsetNumber offno;
 		OffsetNumber maxoffno;
 		bool		updated = false;
+#if PG_VERSION_NUM >= 170000
+		OffsetNumber deletedoffs[MaxOffsetNumber];
+		int			ndeletedoffs = 0;
+#endif
 
 		vacuum_delay_point();
 
@@ -121,6 +135,9 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 			if (!ItemPointerIsValid(&etup->heaptids[0]))
 			{
+#if PG_VERSION_NUM >= 170000
+				deletedoffs[ndeletedoffs++] = offno;
+#else
 				ItemPointerData indextid;
 				bool		found;
 
@@ -129,6 +146,7 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 				tidhash_insert(vacuumstate->deleting, indextid, &found);
 				Assert(!found);
+#endif
 			}
 			else if (etup->level > highestLevel)
 			{
@@ -157,6 +175,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 			}
 		}
 
+#if PG_VERSION_NUM >= 170000
+		TidStoreSetBlockOffsets(vacuumstate->deleting, blkno, deletedoffs, ndeletedoffs);
+#endif
+
 		blkno = HnswPageGetOpaque(page)->nextblkno;
 
 		if (updated)
@@ -756,7 +778,11 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 	HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
 
 	/* Create hash table */
+#if PG_VERSION_NUM >= 170000
+	vacuumstate->deleting = TidStoreCreateLocal((AmAutoVacuumWorkerProcess() && autovacuum_work_mem != -1) ? autovacuum_work_mem : maintenance_work_mem, true);
+#else
 	vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
+#endif
 }
 
 /*
@@ -765,7 +791,11 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 static void
 FreeVacuumState(HnswVacuumState * vacuumstate)
 {
+#if PG_VERSION_NUM >= 170000
+	TidStoreDestroy(vacuumstate->deleting);
+#else
 	tidhash_destroy(vacuumstate->deleting);
+#endif
 	FreeAccessStrategy(vacuumstate->bas);
 	pfree(vacuumstate->ntup);
 	MemoryContextDelete(vacuumstate->tmpCtx);

src/ivfkmeans.c

@@ -8,7 +8,6 @@
 #include "fmgr.h"
 #include "ivfflat.h"
 #include "miscadmin.h"
-#include "utils/datum.h"
 #include "utils/memutils.h"
 #include "utils/relcache.h"
 
@@ -106,44 +105,16 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
 }
 
 /*
- * Check if vector array contains a vector
- */
-static bool
-VectorArrayContains(VectorArray arr, Pointer val)
-{
-	Datum		d = PointerGetDatum(val);
-
-	for (int i = 0; i < arr->length; i++)
-	{
-		if (datumIsEqual(d, PointerGetDatum(VectorArrayGet(arr, i)), false, -1))
-			return true;
-	}
-	return false;
-}
-
-/*
- * Quick approach if we have little data
+ * Quick approach if we have no data
  */
 static void
-QuickCenters(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
+RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
 {
 	int			dimensions = centers->dim;
 	FmgrInfo   *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
 	Oid			collation = index->rd_indcollation[0];
 	float	   *x = (float *) palloc(sizeof(float) * dimensions);
 
-	/* Fill with unique samples (already normalized) */
-	for (int i = 0; i < samples->length; i++)
-	{
-		Pointer		sample = VectorArrayGet(samples, i);
-
-		if (!VectorArrayContains(centers, sample))
-		{
-			VectorArraySet(centers, centers->length, sample);
-			centers->length++;
-		}
-	}
-
 	/* Fill with random data */
 	while (centers->length < centers->maxlen)
 	{
@@ -577,8 +548,8 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
 													ALLOCSET_DEFAULT_SIZES);
 	MemoryContext oldCtx = MemoryContextSwitchTo(kmeansCtx);
 
-	if (samples->length <= centers->maxlen)
-		QuickCenters(index, samples, centers, typeInfo);
+	if (samples->length == 0)
+		RandomCenters(index, centers, typeInfo);
 	else
 		ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);
 

src/ivfutils.c

@@ -23,15 +23,11 @@
 VectorArray
 VectorArrayInit(int maxlen, int dimensions, Size itemsize)
 {
-	VectorArray res;
-
-	if (maxlen < 1 || dimensions < 1 || itemsize == 0)
-		elog(ERROR, "cannot create vector array");
+	VectorArray res = palloc(sizeof(VectorArrayData));
 
 	/* Ensure items are aligned to prevent UB */
 	itemsize = MAXALIGN(itemsize);
 
-	res = palloc(sizeof(VectorArrayData));
 	res->length = 0;
 	res->maxlen = maxlen;
 	res->dim = dimensions;

src/vector.c

@@ -40,7 +40,7 @@
 #endif
 
 #if PG_VERSION_NUM >= 180000
-PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.5");
+PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.4");
 #else
 PG_MODULE_MAGIC;
 #endif

test/t/047_hnsw_vacuum_insert.pl

@@ -31,7 +31,7 @@ $node->pgbench(
     {
         "047_hnsw_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
         "047_hnsw_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
-        "047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
+        "047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
         "047_hnsw_vacuum_insert_vacuum\@1" => "VACUUM tst;"
     }
 );

test/t/048_ivfflat_vacuum_insert.pl

@@ -1,39 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $dim = 3;
-my $array_sql = join(",", ('random()') x $dim);
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table and index
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
-$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
-$node->safe_psql("postgres",
-    "INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
-);
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 10);");
-
-# Test no errors
-$node->pgbench(
-    "--no-vacuum --client=5 --transactions=1500",
-    0,
-    [qr{actually processed}],
-    [qr{^$}],
-    "concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
-    {
-        "048_ivfflat_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
-        "048_ivfflat_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
-        "048_ivfflat_vacuum_insert_select\@500" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
-        "048_ivfflat_vacuum_insert_vacuum\@1" => "VACUUM tst;"
-    }
-);
-
-done_testing();

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat and hnsw access methods'
-default_version = '0.8.5'
+default_version = '0.8.4'
 module_pathname = '$libdir/vector'
 relocatable = true