Added bulk hashing [skip ci]

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.8.0 (2024-10-30)
+## 0.8.0 (unreleased)
 
 - Added support for iterative index scans
 - Added casts for arrays to `sparsevec`

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.8.0",
+	"version": "0.7.4",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -12,7 +12,7 @@
 	"prereqs": {
 		"runtime": {
 			"requires": {
-				"PostgreSQL": "13.0.0"
+				"PostgreSQL": "12.0.0"
 			}
 		}
 	},
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.8.0",
+			"version": "0.7.4",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.8.0
+EXTVERSION = 0.7.4
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*--*.sql)

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.8.0
+EXTVERSION = 0.7.4
 
 DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
 OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

README.md

@@ -17,11 +17,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
 
 ### Linux and Mac
 
-Compile and install the extension (supports Postgres 13+)
+Compile and install the extension (supports Postgres 12+)
 
 ```sh
 cd /tmp
-git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -46,7 +46,7 @@ Then use `nmake` to build:
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\16"
 cd %TEMP%
-git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -324,7 +324,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
 
 ### Indexing Progress
 
-Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
+Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
 
 ```sql
 SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -410,7 +410,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
 
 ### Indexing Progress
 
-Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
+Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
 
 ```sql
 SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -477,7 +477,7 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
 
 ## Iterative Index Scans
 
-*Added in 0.8.0*
+*Unreleased*
 
 With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
 
@@ -1153,7 +1153,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
 ```

src/hnswutils.c

@@ -722,6 +722,8 @@ CountElement(HnswElement skipElement, HnswElement e)
 static void
 HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize)
 {
+	uint32		hashes[HNSW_MAX_M * 2];
+
 	/* Get the neighborhood at layer lc */
 	HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc);
 
@@ -733,14 +735,33 @@ HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unv
 	*unvisitedLength = 0;
 
 	for (int i = 0; i < localNeighborhood->length; i++)
+		hashes[i] = HnswPtrAccess(base, localNeighborhood->items[i].element)->hash;
+
+	if (base != NULL)
 	{
-		HnswCandidate *hc = &localNeighborhood->items[i];
+		for (int i = 0; i < localNeighborhood->length; i++)
-		bool		found;
+		{
+			HnswCandidate *hc = &localNeighborhood->items[i];
+			bool		found;
 
-		AddToVisited(base, v, hc->element, true, &found);
+			offsethash_insert_hash(v->offsets, HnswPtrOffset(hc->element), hashes[i], &found);
 
-		if (!found)
+			if (!found)
-			unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
+				unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
+		}
+	}
+	else
+	{
+		for (int i = 0; i < localNeighborhood->length; i++)
+		{
+			HnswCandidate *hc = &localNeighborhood->items[i];
+			bool		found;
+
+			pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), hashes[i], &found);
+
+			if (!found)
+				unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
+		}
 	}
 }
 

src/sparsevec.c

@@ -4,7 +4,6 @@
 #include <math.h>
 
 #include "catalog/pg_type.h"
-#include "common/shortest_dec.h"
 #include "common/string.h"
 #include "fmgr.h"
 #include "halfutils.h"
@@ -13,10 +12,17 @@
 #include "sparsevec.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
-#include "utils/float.h"
 #include "utils/lsyscache.h"
 #include "vector.h"
 
+#if PG_VERSION_NUM >= 120000
+#include "common/shortest_dec.h"
+#include "utils/float.h"
+#else
+#include <float.h>
+#include "utils/builtins.h"
+#endif
+
 typedef struct SparseInputElement
 {
 	int32		index;

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat and hnsw access methods'
-default_version = '0.8.0'
+default_version = '0.7.4'
 module_pathname = '$libdir/vector'
 relocatable = true