Added tuple stats for HNSW [skip ci]

CHANGELOG.md

@@ -1,6 +1,5 @@
-## 0.7.3 (2024-07-22)
+## 0.7.3 (unreleased)
 
-- Fixed `failed to add index item` error with `sparsevec`
 - Fixed compilation error with FreeBSD ARM
 - Fixed compilation warning with MSVC and Postgres 16
 

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.7.3",
+	"version": "0.7.2",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.7.3",
+			"version": "0.7.2",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.7.3
+EXTVERSION = 0.7.2
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.7.3
+EXTVERSION = 0.7.2
 
 OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
 HEADERS = src\halfvec.h src\sparsevec.h src\vector.h

README.md

@@ -21,7 +21,7 @@ Compile and install the extension (supports Postgres 12+)
 
 ```sh
 cd /tmp
-git clone --branch v0.7.3 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.2 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -46,7 +46,7 @@ Then use `nmake` to build:
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\16"
 cd %TEMP%
-git clone --branch v0.7.3 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.2 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -1051,7 +1051,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.7.3 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.2 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build --pull --build-arg PG_MAJOR=16 -t myuser/pgvector .
 ```

sql/vector--0.7.2--0.7.3.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.3'" to load this file. \quit

src/hnsw.c

@@ -20,6 +20,10 @@ int			hnsw_ef_search;
 int			hnsw_lock_tranche_id;
 static relopt_kind hnsw_relopt_kind;
 
+#ifdef HNSW_STATS
+int			hnsw_tuples;
+#endif
+
 /*
  * Assign a tranche ID for our LWLocks. This only needs to be done by one
  * backend, as the tranche ID is remembered in shared memory.
@@ -135,6 +139,10 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 
 	genericcostestimate(root, path, loop_count, &costs);
 
+#ifdef HNSW_STATS
+	elog(INFO, "estimated tuples = %0.f", costs.numIndexTuples);
+#endif
+
 	/* Use total cost since most work happens before first tuple is returned */
 	*indexStartupCost = costs.indexTotalCost;
 	*indexTotalCost = costs.indexTotalCost;

src/hnsw.h

@@ -113,6 +113,10 @@
 extern int	hnsw_ef_search;
 extern int	hnsw_lock_tranche_id;
 
+#ifdef HNSW_STATS
+extern int	hnsw_tuples;
+#endif
+
 typedef struct HnswElementData HnswElementData;
 typedef struct HnswNeighborArray HnswNeighborArray;
 

src/hnswinsert.c

@@ -36,15 +36,14 @@ GetInsertPage(Relation index)
  * Check for a free offset
  */
 static bool
-HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
+HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
 {
 	OffsetNumber offno;
 	OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
 
 	for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
 	{
-		ItemId		eitemid = PageGetItemId(page, offno);
+		HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
-		HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, eitemid);
 
 		/* Skip neighbor tuples */
 		if (!HnswIsElementTuple(etup))
@@ -55,9 +54,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
 			BlockNumber elementPage = BufferGetBlockNumber(buf);
 			BlockNumber neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
 			OffsetNumber neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
-			ItemId		nitemid;
+			ItemId		itemid;
-			Size		pageFree;
-			Size		npageFree;
 
 			if (!BlockNumberIsValid(*newInsertPage))
 				*newInsertPage = elementPage;
@@ -76,25 +73,10 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
 				*npage = BufferGetPage(*nbuf);
 			}
 
-			nitemid = PageGetItemId(*npage, neighborOffno);
+			itemid = PageGetItemId(*npage, neighborOffno);
 
-			/* Ensure aligned for space check */
+			/* Check for space on neighbor tuple page */
-			Assert(etupSize == MAXALIGN(etupSize));
+			if (PageGetFreeSpace(*npage) + ItemIdGetLength(itemid) - sizeof(ItemIdData) >= ntupSize)
-			Assert(ntupSize == MAXALIGN(ntupSize));
-
-			/*
-			 * Calculate free space individually since tuples are overwritten
-			 * individually (in separate calls to PageIndexTupleOverwrite)
-			 */
-			pageFree = ItemIdGetLength(eitemid) + PageGetExactFreeSpace(page);
-			npageFree = ItemIdGetLength(nitemid);
-			if (neighborPage != elementPage)
-				npageFree += PageGetExactFreeSpace(*npage);
-			else if (pageFree >= etupSize)
-				npageFree += pageFree - etupSize;
-
-			/* Check for space */
-			if (pageFree >= etupSize && npageFree >= ntupSize)
 			{
 				*freeOffno = offno;
 				*freeNeighborOffno = neighborOffno;
@@ -202,7 +184,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 		}
 
 		/* Next, try space from a deleted element */
-		if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
+		if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
 		{
 			if (nbuf != buf)
 			{

src/hnswscan.c

@@ -31,13 +31,23 @@ GetScanItems(IndexScanDesc scan, Datum q)
 
 	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
 
+#ifdef HNSW_STATS
+	hnsw_tuples = 1;
+#endif
+
 	for (int lc = entryPoint->level; lc >= 1; lc--)
 	{
 		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
 		ep = w;
 	}
 
-	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
+	w = HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
+
+#ifdef HNSW_STATS
+	elog(INFO, "total tuples = %d", hnsw_tuples);
+#endif
+
+	return w;
 }
 
 /*

src/hnswutils.c

@@ -801,6 +801,11 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 				HnswElement eElement = HnswPtrAccess(base, e->element);
 				bool		alwaysAdd = wlen < ef;
 
+#ifdef HNSW_STATS
+				if (!inserting)
+					hnsw_tuples++;
+#endif
+
 				f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
 
 				if (index == NULL)

test/t/038_hnsw_sparsevec_vacuum_insert.pl

@@ -1,38 +0,0 @@
-use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
-use Test::More;
-
-# Initialize node
-my $node = get_new_node('node');
-$node->init;
-$node->start;
-
-# Create table and index
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v sparsevec(100000));");
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v sparsevec_l2_ops);");
-
-for (1 .. 3) {
-    for (1 .. 100) {
-        my @elements;
-        my %indices;
-        for (1 .. int(rand() * 100)) {
-            my $index = int(rand() * (100000 - 1)) + 1;
-            if (!exists($indices{$index})) {
-                my $value = rand();
-                push(@elements, "$index:$value");
-                $indices{$index} = 1;
-            }
-        }
-        my $embedding = "{" . join(",", @elements) . "}/100000";
-        $node->safe_psql("postgres", "INSERT INTO tst (v) VALUES ('$embedding');");
-    }
-
-    $node->safe_psql("postgres", "DELETE FROM tst WHERE i % 2 = 0;");
-    $node->safe_psql("postgres", "VACUUM tst;");
-    is(1, 1);
-}
-
-done_testing();

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat and hnsw access methods'
-default_version = '0.7.3'
+default_version = '0.7.2'
 module_pathname = '$libdir/vector'
 relocatable = true