Added HnswSupport struct [skip ci]

These places called PageIndexTupleOverwrite(), with the new tuple
pointing directly to the original page. The PageIndexTupleOverwrite()
call is unnecessary in these cases, as we have already modified the
tuple on the page directly. Moreover, PageIndexTupleOverWrite() will
call memcpy with same src and dst arguments, which is undefined
behavior.

It's OK to modify the pages on disk directly, and we don't need
critical sections, because we either use the generic xlog functions
which create a temporary copy of the page, or we are building a new
index so if we crash the whole index is invisible and will be dropped
anyway.

.github/workflows/build.yml

@@ -57,9 +57,9 @@ jobs:
       - run: |
           brew install cpanm
           cpanm --notest IPC::Run
-          wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
+          wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
-          tar xf REL_14_5.tar.gz
+          tar xf REL_14_10.tar.gz
-      - run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
+      - run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
       - run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make PG_CFLAGS="-DUSE_ASSERT_CHECKING"
   windows:
     runs-on: windows-latest

CHANGELOG.md

@@ -1,16 +1,21 @@
-## 0.6.0 (unreleased)
+## 0.6.1 (unreleased)
 
-If upgrading with Postgres < 13, see [this note](https://github.com/pgvector/pgvector#060).
+- Fixed error with `ANALYZE` and vectors with different dimensions
 
+## 0.6.0 (2024-01-29)
+
+If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060).
+
+- Added support for parallel index builds for HNSW
+- Added validation for GUC parameters
 - Changed storage for vector from `extended` to `external`
 - Improved performance of HNSW
-- Added support for parallel index builds for HNSW
-- Added `hamming_distance` function
-- Added validation for GUC parameters
 - Reduced memory usage for HNSW index builds
 - Reduced WAL generation for HNSW index builds
 - Fixed error with logical replication
-- Fixed `invalid memory alloc request size` error with HNSW index build
+- Fixed `invalid memory alloc request size` error with HNSW index builds
+- Moved Docker image to `pgvector` org
+- Added Docker tags for each supported version of Postgres
 - Dropped support for Postgres 11
 
 ## 0.5.1 (2023-10-10)
@@ -59,7 +64,7 @@ If upgrading with Postgres < 13, see [this note](https://github.com/pgvector/pgv
 
 ## 0.4.0 (2023-01-11)
 
-If upgrading with Postgres < 13, see [this note](https://github.com/pgvector/pgvector#040).
+If upgrading with Postgres < 13, see [this note](https://github.com/pgvector/pgvector/blob/v0.4.0/README.md#040).
 
 - Changed text representation for vector elements to match `real`
 - Changed storage for vector from `plain` to `extended`
@@ -76,7 +81,7 @@ If upgrading with Postgres < 13, see [this note](https://github.com/pgvector/pgv
 
 ## 0.3.1 (2022-11-02)
 
-If upgrading from 0.2.7 or 0.3.0, [recreate](https://github.com/pgvector/pgvector#031) all `ivfflat` indexes after upgrading to ensure all data is indexed.
+If upgrading from 0.2.7 or 0.3.0, [recreate](https://github.com/pgvector/pgvector/blob/v0.3.1/README.md#031) all `ivfflat` indexes after upgrading to ensure all data is indexed.
 
 - Fixed issue with inserts silently corrupting `ivfflat` indexes (introduced in 0.2.7)
 - Fixed segmentation fault with index creation when lists > 6500

Dockerfile

@@ -1,4 +1,4 @@
-ARG PG_MAJOR=15
+ARG PG_MAJOR=16
 FROM postgres:$PG_MAJOR
 ARG PG_MAJOR
 

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.5.1",
+			"version": "0.6.0",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.5.1
+EXTVERSION = 0.6.0
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)
@@ -65,13 +65,17 @@ dist:
 	mkdir -p dist
 	git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master
 
+# for Docker
+PG_MAJOR ?= 16
+
 .PHONY: docker
 
 docker:
-	docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest .
+	docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
+	docker build --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
 
 .PHONY: docker-release
 
 docker-release:
-	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 -t ankane/pgvector:latest .
+	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
-	docker buildx build --push --platform linux/amd64,linux/arm64 -t ankane/pgvector:v$(EXTVERSION) .
+	docker buildx build --push --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.5.1
+EXTVERSION = 0.6.0
 
 OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
 HEADERS = src\vector.h

README.md

@@ -14,11 +14,13 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
 
 ## Installation
 
-Compile and install the extension (supports Postgres 11+)
+### Linux and Mac
+
+Compile and install the extension (supports Postgres 12+)
 
 ```sh
 cd /tmp
-git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -28,6 +30,28 @@ See the [installation notes](#installation-notes) if you run into issues
 
 You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
 
+### Windows
+
+Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
+
+```cmd
+call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+```
+
+Note: The exact path will vary depending on your Visual Studio version and edition
+
+Then use `nmake` to build:
+
+```cmd
+set "PGROOT=C:\Program Files\PostgreSQL\16"
+git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
+cd pgvector
+nmake /F Makefile.win
+nmake /F Makefile.win install
+```
+
+You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
+
 ## Getting Started
 
 Enable the extension (do this once in each database where you want to use it)
@@ -240,6 +264,14 @@ HINT:  Increase maintenance_work_mem to speed up builds.
 
 Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on the server
 
+Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
+
+```sql
+SET max_parallel_maintenance_workers = 7; -- plus leader
+```
+
+For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
+
 ### Indexing Progress
 
 Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
@@ -398,6 +430,15 @@ To speed up queries with an IVFFlat index, increase the number of inverted lists
 CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
 ```
 
+## Vacuuming
+
+Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
+
+```sql
+REINDEX INDEX CONCURRENTLY index_name;
+VACUUM table_name;
+```
+
 ## Languages
 
 Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
@@ -615,44 +656,24 @@ Note: Replace `16` with your Postgres server version
 
 If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
 
-### Windows
-
-Support for Windows is currently experimental. Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
-
-```cmd
-call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
-```
-
-Note: The exact path will vary depending on your Visual Studio version and edition
-
-Then use `nmake` to build:
-
-```cmd
-set "PGROOT=C:\Program Files\PostgreSQL\16"
-git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
-cd pgvector
-nmake /F Makefile.win
-nmake /F Makefile.win install
-```
-
 ## Additional Installation Methods
 
 ### Docker
 
-Get the [Docker image](https://hub.docker.com/r/ankane/pgvector) with:
+Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
 
 ```sh
-docker pull ankane/pgvector
+docker pull pgvector/pgvector:pg16
 ```
 
-This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (run it the same way).
+This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
 
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
 cd pgvector
-docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
+docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
 ```
 
 ### Homebrew
@@ -729,9 +750,11 @@ SELECT extversion FROM pg_extension WHERE extname = 'vector';
 
 ## Upgrade Notes
 
-### 0.6.0 [unreleased]
+### 0.6.0
 
-If upgrading with Postgres < 13, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
+#### Postgres 12
+
+If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
 
 ```sql
 ALTER TYPE vector SET (STORAGE = external);
@@ -739,28 +762,20 @@ ALTER TYPE vector SET (STORAGE = external);
 
 Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
 
-### 0.4.0
+#### Docker
 
-If upgrading with Postgres < 13, remove this line from `sql/vector--0.3.2--0.4.0.sql`:
+The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
 
-```sql
+```sh
-ALTER TYPE vector SET (STORAGE = extended);
+docker pull pgvector/pgvector:pg16
+# or
+docker pull pgvector/pgvector:0.6.0-pg16
 ```
 
-Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
+Also, if you’ve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
 
-### 0.3.1
+```sh
-
+docker run --shm-size=1g ...
-If upgrading from 0.2.7 or 0.3.0, recreate all `ivfflat` indexes after upgrading to ensure all data is indexed.
-
-```sql
--- Postgres 12+
-REINDEX INDEX CONCURRENTLY index_name;
-
--- Postgres < 12
-CREATE INDEX CONCURRENTLY temp_name ON table USING ivfflat (column opclass);
-DROP INDEX CONCURRENTLY index_name;
-ALTER INDEX temp_name RENAME TO index_name;
 ```
 
 ## Thanks
@@ -806,8 +821,8 @@ make prove_installcheck  # TAP tests
 To run single tests:
 
 ```sh
-make installcheck REGRESS=functions                    # regression test
+make installcheck REGRESS=functions                            # regression test
-make prove_installcheck PROVE_TESTS=test/t/001_wal.pl  # TAP test
+make prove_installcheck PROVE_TESTS=test/t/001_ivfflat_wal.pl  # TAP test
 ```
 
 To enable benchmarking:

sql/vector--0.5.1--0.6.0.sql

@@ -3,6 +3,3 @@
 
 -- remove this single line for Postgres < 13
 ALTER TYPE vector SET (STORAGE = external);
-
-CREATE FUNCTION hamming_distance(bytea, bytea) RETURNS float8
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;

sql/vector.sql

@@ -290,8 +290,3 @@ CREATE OPERATOR CLASS vector_cosine_ops
 	OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops,
 	FUNCTION 1 vector_negative_inner_product(vector, vector),
 	FUNCTION 2 vector_norm(vector);
-
--- bytea functions
-
-CREATE FUNCTION hamming_distance(bytea, bytea) RETURNS float8
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;

src/hnsw.h

@@ -60,10 +60,13 @@
 #define PROGRESS_HNSW_PHASE_LOAD		2
 
 #define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData))
+#define HNSW_TUPLE_ALLOC_SIZE BLCKSZ
 
 #define HNSW_ELEMENT_TUPLE_SIZE(size)	MAXALIGN(offsetof(HnswElementTupleData, data) + (size))
 #define HNSW_NEIGHBOR_TUPLE_SIZE(level, m)	MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
 
+#define HNSW_NEIGHBOR_ARRAY_SIZE(lm)	(offsetof(HnswNeighborArray, items) + sizeof(HnswCandidate) * (lm))
+
 #define HnswPageGetOpaque(page)	((HnswPageOpaque) PageGetSpecialPointer(page))
 #define HnswPageGetMeta(page)	((HnswMetaPageData *) PageGetContents(page))
 
@@ -231,6 +234,12 @@ typedef struct HnswAllocator
 	void	   *state;
 }			HnswAllocator;
 
+typedef struct HnswSupport
+{
+	FmgrInfo   *procinfo;
+	Oid			collation;
+}			HnswSupport;
+
 typedef struct HnswBuildState
 {
 	/* Info */
@@ -249,7 +258,7 @@ typedef struct HnswBuildState
 	double		reltuples;
 
 	/* Support functions */
-	FmgrInfo   *procinfo;
+	HnswSupport support;
 	FmgrInfo   *normprocinfo;
 	Oid			collation;
 
@@ -326,7 +335,7 @@ typedef struct HnswScanOpaqueData
 	MemoryContext tmpCtx;
 
 	/* Support functions */
-	FmgrInfo   *procinfo;
+	HnswSupport support;
 	FmgrInfo   *normprocinfo;
 	Oid			collation;
 }			HnswScanOpaqueData;
@@ -346,8 +355,7 @@ typedef struct HnswVacuumState
 	int			efConstruction;
 
 	/* Support functions */
-	FmgrInfo   *procinfo;
+	HnswSupport support;
-	Oid			collation;
 
 	/* Variables */
 	struct tidhash_hash *deleted;
@@ -363,28 +371,29 @@ typedef struct HnswVacuumState
 int			HnswGetM(Relation index);
 int			HnswGetEfConstruction(Relation index);
 FmgrInfo   *HnswOptionalProcInfo(Relation index, uint16 procnum);
+void		HnswInitSupport(HnswSupport * support, Relation index);
 bool		HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
 Buffer		HnswNewBuffer(Relation index, ForkNumber forkNum);
 void		HnswInitPage(Buffer buf, Page page);
 void		HnswInit(void);
-List	   *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
+List	   *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement);
 HnswElement HnswGetEntryPoint(Relation index);
 void		HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
 void	   *HnswAlloc(HnswAllocator * allocator, Size size);
 HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
 HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
-void		HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
+void		HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
-HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
+HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, HnswSupport * support, bool loadVec);
 void		HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
 void		HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
 void		HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
 void		HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * alloc);
-bool		HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel, bool building);
+bool		HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
-void		HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
+void		HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e, int m, bool checkExisting, bool building);
 void		HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
-void		HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
+void		HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, HnswSupport * support, bool loadVec);
 void		HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
-void		HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
+void		HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, HnswSupport * support);
 void		HnswLoadNeighbors(HnswElement element, Relation index, int m);
 PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
 

src/hnswbuild.c

@@ -65,8 +65,6 @@
 #define CALLBACK_ITEM_POINTER HeapTuple hup
 #endif
 
-#define UpdateProgress(index, val) pgstat_progress_update_param(index, val)
-
 #if PG_VERSION_NUM >= 140000
 #include "utils/backend_status.h"
 #include "utils/wait_event.h"
@@ -143,14 +141,13 @@ HnswBuildAppendPage(Relation index, Buffer *buf, Page *page, ForkNumber forkNum)
 }
 
 /*
- * Create element pages
+ * Create graph pages
  */
 static void
-CreateElementPages(HnswBuildState * buildstate)
+CreateGraphPages(HnswBuildState * buildstate)
 {
 	Relation	index = buildstate->index;
 	ForkNumber	forkNum = buildstate->forkNum;
-	Size		etupAllocSize;
 	Size		maxSize;
 	HnswElementTuple etup;
 	HnswNeighborTuple ntup;
@@ -162,12 +159,11 @@ CreateElementPages(HnswBuildState * buildstate)
 	char	   *base = buildstate->hnswarea;
 
 	/* Calculate sizes */
-	etupAllocSize = BLCKSZ;
 	maxSize = HNSW_MAX_SIZE;
 
 	/* Allocate once */
-	etup = palloc0(etupAllocSize);
+	etup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
-	ntup = palloc0(BLCKSZ);
+	ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
 
 	/* Prepare first page */
 	buf = HnswNewBuffer(index, forkNum);
@@ -186,7 +182,7 @@ CreateElementPages(HnswBuildState * buildstate)
 		iter = element->next;
 
 		/* Zero memory for each element */
-		MemSet(etup, 0, etupAllocSize);
+		MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE);
 
 		/* Calculate sizes */
 		etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(valuePtr));
@@ -194,7 +190,7 @@ CreateElementPages(HnswBuildState * buildstate)
 		combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
 
 		/* Initial size check */
-		if (etupSize > etupAllocSize)
+		if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
 			elog(ERROR, "index tuple too large");
 
 		HnswSetElementTuple(base, etup, element);
@@ -246,10 +242,10 @@ CreateElementPages(HnswBuildState * buildstate)
 }
 
 /*
- * Create neighbor pages
+ * Write neighbor tuples
  */
 static void
-CreateNeighborPages(HnswBuildState * buildstate)
+WriteNeighborTuples(HnswBuildState * buildstate)
 {
 	Relation	index = buildstate->index;
 	ForkNumber	forkNum = buildstate->forkNum;
@@ -259,29 +255,32 @@ CreateNeighborPages(HnswBuildState * buildstate)
 	HnswNeighborTuple ntup;
 
 	/* Allocate once */
-	ntup = palloc0(BLCKSZ);
+	ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
 
 	while (!HnswPtrIsNull(base, iter))
 	{
-		HnswElement e = HnswPtrAccess(base, iter);
+		HnswElement element = HnswPtrAccess(base, iter);
 		Buffer		buf;
 		Page		page;
-		Size		ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m);
+		Size		ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
 
 		/* Update iterator */
-		iter = e->next;
+		iter = element->next;
+
+		/* Zero memory for each element */
+		MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
 
 		/* Can take a while, so ensure we can interrupt */
 		/* Needs to be called when no buffer locks are held */
 		CHECK_FOR_INTERRUPTS();
 
-		buf = ReadBufferExtended(index, forkNum, e->neighborPage, RBM_NORMAL, NULL);
+		buf = ReadBufferExtended(index, forkNum, element->neighborPage, RBM_NORMAL, NULL);
 		LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 		page = BufferGetPage(buf);
 
-		HnswSetNeighborTuple(base, ntup, e, m);
+		HnswSetNeighborTuple(base, ntup, element, m);
 
-		if (!PageIndexTupleOverwrite(page, e->neighborOffno, (Item) ntup, ntupSize))
+		if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
 			elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
 
 		/* Commit */
@@ -303,8 +302,8 @@ FlushPages(HnswBuildState * buildstate)
 #endif
 
 	CreateMetaPage(buildstate);
-	CreateElementPages(buildstate);
+	CreateGraphPages(buildstate);
-	CreateNeighborPages(buildstate);
+	WriteNeighborTuples(buildstate);
 
 	buildstate->graph->flushed = true;
 	MemoryContextReset(buildstate->graphCtx);
@@ -374,7 +373,7 @@ AddElementInMemory(char *base, HnswGraph * graph, HnswElement element)
  * Update neighbors
  */
 static void
-UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswElement e, int m)
+UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
 {
 	for (int lc = e->level; lc >= 0; lc--)
 	{
@@ -391,7 +390,7 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
 
 			/* Use element for lock instead of hc since hc can be replaced */
 			LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
-			HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
+			HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, support);
 			LWLockRelease(&neighborElement->lock);
 		}
 	}
@@ -401,7 +400,7 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
  * Update graph in memory
  */
 static void
-UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
+UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
 {
 	HnswGraph  *graph = buildstate->graph;
 	char	   *base = buildstate->hnswarea;
@@ -414,7 +413,7 @@ UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int
 	AddElementInMemory(base, graph, element);
 
 	/* Update neighbors */
-	UpdateNeighborsInMemory(base, procinfo, collation, element, m);
+	UpdateNeighborsInMemory(base, support, element, m);
 
 	/* Update entry point if needed (already have lock) */
 	if (entryPoint == NULL || element->level > entryPoint->level)
@@ -427,8 +426,6 @@ UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int
 static void
 InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
 {
-	FmgrInfo   *procinfo = buildstate->procinfo;
-	Oid			collation = buildstate->collation;
 	HnswGraph  *graph = buildstate->graph;
 	HnswElement entryPoint;
 	LWLock	   *entryLock = &graph->entryLock;
@@ -454,10 +451,10 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
 	}
 
 	/* Find neighbors for element */
-	HnswFindElementNeighbors(base, element, entryPoint, NULL, procinfo, collation, m, efConstruction, false);
+	HnswFindElementNeighbors(base, element, entryPoint, NULL, &buildstate->support, m, efConstruction, false);
 
 	/* Update graph in memory */
-	UpdateGraphInMemory(procinfo, collation, element, m, efConstruction, entryPoint, buildstate);
+	UpdateGraphInMemory(&buildstate->support, element, m, efConstruction, entryPoint, buildstate);
 
 	/* Release entry lock */
 	LWLockRelease(entryLock);
@@ -498,7 +495,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	{
 		LWLockRelease(flushLock);
 
-		return HnswInsertTupleOnDisk(index, value, values, isnull, heaptid, buildstate->heap, true);
+		return HnswInsertTupleOnDisk(index, value, values, isnull, heaptid, true);
 	}
 
 	/*
@@ -530,7 +527,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 
 		LWLockRelease(flushLock);
 
-		return HnswInsertTupleOnDisk(index, value, values, isnull, heaptid, buildstate->heap, true);
+		return HnswInsertTupleOnDisk(index, value, values, isnull, heaptid, true);
 	}
 
 	/* Ok, we can proceed to allocate the element */
@@ -587,7 +584,7 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
 	{
 		/* Update progress */
 		SpinLockAcquire(&graph->lock);
-		UpdateProgress(PROGRESS_CREATEIDX_TUPLES_DONE, ++graph->indtuples);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, ++graph->indtuples);
 		SpinLockRelease(&graph->lock);
 	}
 
@@ -684,7 +681,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
 	buildstate->indtuples = 0;
 
 	/* Get support functions */
-	buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+	HnswInitSupport(&buildstate->support, index);
 	buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
 	buildstate->collation = index->rd_indcollation[0];
 
@@ -1048,7 +1045,7 @@ BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
 {
 	int			parallel_workers = 0;
 
-	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_HNSW_PHASE_LOAD);
+	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_HNSW_PHASE_LOAD);
 
 	/* Calculate parallel workers */
 	if (buildstate->heap != NULL)

src/hnswinsert.c

@@ -340,7 +340,7 @@ ConnectionExists(HnswElement e, HnswNeighborTuple ntup, int startIdx, int lm)
  * Update neighbors
  */
 void
-HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building)
+HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e, int m, bool checkExisting, bool building)
 {
 	char	   *base = NULL;
 
@@ -355,9 +355,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
 			Buffer		buf;
 			Page		page;
 			GenericXLogState *state;
-			ItemId		itemid;
 			HnswNeighborTuple ntup;
-			Size		ntupSize;
 			int			idx = -1;
 			int			startIdx;
 			HnswElement neighborElement = HnswPtrAccess(base, hc->element);
@@ -375,7 +373,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
 			 */
 
 			/* Select neighbors */
-			HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, procinfo, collation);
+			HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, support);
 
 			/* New element was not selected as a neighbor */
 			if (idx == -1)
@@ -396,9 +394,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
 			}
 
 			/* Get tuple */
-			itemid = PageGetItemId(page, offno);
+			ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
-			ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
-			ntupSize = ItemIdGetLength(itemid);
 
 			/* Calculate index for update */
 			startIdx = (neighborElement->level - lc) * m;
@@ -427,13 +423,9 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
 			{
 				ItemPointer indextid = &ntup->indextids[idx];
 
-				/* Update neighbor */
+				/* Update neighbor on the buffer */
 				ItemPointerSet(indextid, e->blkno, e->offno);
 
-				/* Overwrite tuple */
-				if (!PageIndexTupleOverwrite(page, offno, (Item) ntup, ntupSize))
-					elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
-
 				/* Commit */
 				if (building)
 					MarkBufferDirty(buf);
@@ -457,9 +449,7 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
 	Buffer		buf;
 	Page		page;
 	GenericXLogState *state;
-	ItemId		itemid;
 	HnswElementTuple etup;
-	Size		etupSize;
 	int			i;
 
 	/* Read page */
@@ -477,9 +467,7 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
 	}
 
 	/* Find space */
-	itemid = PageGetItemId(page, dup->offno);
+	etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, dup->offno));
-	etup = (HnswElementTuple) PageGetItem(page, itemid);
-	etupSize = ItemIdGetLength(itemid);
 	for (i = 0; i < HNSW_HEAPTIDS; i++)
 	{
 		if (!ItemPointerIsValid(&etup->heaptids[i]))
@@ -495,13 +483,9 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
 		return false;
 	}
 
-	/* Add heap TID */
+	/* Add heap TID, modifying the tuple on the page directly */
 	etup->heaptids[i] = element->heaptids[0];
 
-	/* Overwrite tuple */
-	if (!PageIndexTupleOverwrite(page, dup->offno, (Item) etup, etupSize))
-		elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
-
 	/* Commit */
 	if (building)
 		MarkBufferDirty(buf);
@@ -543,7 +527,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
  * Update graph on disk
  */
 static void
-UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
+UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
 {
 	BlockNumber newInsertPage = InvalidBlockNumber;
 
@@ -559,7 +543,7 @@ UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement
 		HnswUpdateMetaPage(index, 0, NULL, newInsertPage, MAIN_FORKNUM, building);
 
 	/* Update neighbors */
-	HnswUpdateNeighborsOnDisk(index, procinfo, collation, element, m, false, building);
+	HnswUpdateNeighborsOnDisk(index, support, element, m, false, building);
 
 	/* Update entry point if needed */
 	if (entryPoint == NULL || element->level > entryPoint->level)
@@ -570,16 +554,17 @@ UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement
  * Insert a tuple into the index
  */
 bool
-HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel, bool building)
+HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building)
 {
 	HnswElement entryPoint;
 	HnswElement element;
 	int			m;
 	int			efConstruction = HnswGetEfConstruction(index);
-	FmgrInfo   *procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
-	Oid			collation = index->rd_indcollation[0];
 	LOCKMODE	lockmode = ShareLock;
 	char	   *base = NULL;
+	HnswSupport support;
+
+	HnswInitSupport(&support, index);
 
 	/*
 	 * Get a shared lock. This allows vacuum to ensure no in-flight inserts
@@ -610,10 +595,10 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
 	}
 
 	/* Find neighbors for element */
-	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false);
+	HnswFindElementNeighbors(base, element, entryPoint, index, &support, m, efConstruction, false);
 
 	/* Update graph on disk */
-	UpdateGraphOnDisk(index, procinfo, collation, element, m, efConstruction, entryPoint, building);
+	UpdateGraphOnDisk(index, &support, element, m, efConstruction, entryPoint, building);
 
 	/* Release lock */
 	UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);
@@ -625,7 +610,7 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
  * Insert a tuple into the index
  */
 static void
-HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
+HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
 {
 	Datum		value;
 	FmgrInfo   *normprocinfo;
@@ -642,7 +627,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
 			return;
 	}
 
-	HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, heapRel, false);
+	HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false);
 }
 
 /*
@@ -671,7 +656,7 @@ hnswinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
 	oldCtx = MemoryContextSwitchTo(insertCtx);
 
 	/* Insert tuple */
-	HnswInsertTuple(index, values, isnull, heap_tid, heap);
+	HnswInsertTuple(index, values, isnull, heap_tid);
 
 	/* Delete memory context */
 	MemoryContextSwitchTo(oldCtx);

src/hnswscan.c

@@ -15,8 +15,6 @@ GetScanItems(IndexScanDesc scan, Datum q)
 {
 	HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
 	Relation	index = scan->indexRelation;
-	FmgrInfo   *procinfo = so->procinfo;
-	Oid			collation = so->collation;
 	List	   *ep;
 	List	   *w;
 	int			m;
@@ -29,15 +27,15 @@ GetScanItems(IndexScanDesc scan, Datum q)
 	if (entryPoint == NULL)
 		return NIL;
 
-	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
+	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, &so->support, false));
 
 	for (int lc = entryPoint->level; lc >= 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
+		w = HnswSearchLayer(base, q, ep, 1, lc, index, &so->support, m, false, NULL);
 		ep = w;
 	}
 
-	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
+	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, &so->support, m, false, NULL);
 }
 
 /*
@@ -108,7 +106,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
 									   ALLOCSET_DEFAULT_SIZES);
 
 	/* Set support functions */
-	so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+	HnswInitSupport(&so->support, index);
 	so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
 	so->collation = index->rd_indcollation[0];
 
@@ -202,6 +200,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 		MemoryContextSwitchTo(oldCtx);
 
 		scan->xs_heaptid = *heaptid;
+		scan->xs_recheck = false;
 		scan->xs_recheckorderby = false;
 		return true;
 	}

src/hnswutils.c

@@ -7,6 +7,7 @@
 #include "lib/pairingheap.h"
 #include "storage/bufmgr.h"
 #include "utils/datum.h"
+#include "utils/memdebug.h"
 #include "utils/rel.h"
 #include "vector.h"
 
@@ -148,6 +149,16 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
 	return index_getprocinfo(index, 1, procnum);
 }
 
+/*
+ * Init support function
+ */
+void
+HnswInitSupport(HnswSupport * support, Relation index)
+{
+	support->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+	support->collation = index->rd_indcollation[0];
+}
+
 /*
  * Divide by the norm
  *
@@ -202,6 +213,19 @@ HnswInitPage(Buffer buf, Page page)
 	HnswPageGetOpaque(page)->page_id = HNSW_PAGE_ID;
 }
 
+/*
+ * Allocate a neighbor array
+ */
+static HnswNeighborArray *
+HnswInitNeighborArray(int lm, HnswAllocator * allocator)
+{
+	HnswNeighborArray *a = HnswAlloc(allocator, HNSW_NEIGHBOR_ARRAY_SIZE(lm));
+
+	a->length = 0;
+	a->closerSet = false;
+	return a;
+}
+
 /*
  * Allocate neighbors
  */
@@ -209,22 +233,12 @@ void
 HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * allocator)
 {
 	int			level = element->level;
-
 	HnswNeighborArrayPtr *neighborList = (HnswNeighborArrayPtr *) HnswAlloc(allocator, sizeof(HnswNeighborArrayPtr) * (level + 1));
 
 	HnswPtrStore(base, element->neighbors, neighborList);
 
 	for (int lc = 0; lc <= level; lc++)
-	{
+		HnswPtrStore(base, neighborList[lc], HnswInitNeighborArray(HnswGetLayerM(m, lc), allocator));
-		HnswNeighborArray *a;
-		int			lm = HnswGetLayerM(m, lc);
-
-		HnswPtrStore(base, neighborList[lc], (HnswNeighborArray *) HnswAlloc(allocator, offsetof(HnswNeighborArray, items) + sizeof(HnswCandidate) * lm));
-
-		a = HnswGetNeighbors(base, element, lc);
-		a->length = 0;
-		a->closerSet = false;
-	}
 }
 
 /*
@@ -312,7 +326,10 @@ HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint)
 	if (entryPoint != NULL)
 	{
 		if (BlockNumberIsValid(metap->entryBlkno))
+		{
 			*entryPoint = HnswInitElementFromBlock(metap->entryBlkno, metap->entryOffno);
+			(*entryPoint)->level = metap->entryLevel;
+		}
 		else
 			*entryPoint = NULL;
 	}
@@ -548,7 +565,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
  * Load an element and optionally get its distance from q
  */
 void
-HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
+HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, HnswSupport * support, bool loadVec)
 {
 	Buffer		buf;
 	Page		page;
@@ -568,7 +585,7 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
 
 	/* Calculate distance */
 	if (distance != NULL)
-		*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
+		*distance = (float) DatumGetFloat8(FunctionCall2Coll(support->procinfo, support->collation, *q, PointerGetDatum(&etup->data)));
 
 	UnlockReleaseBuffer(buf);
 }
@@ -577,27 +594,27 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
  * Get the distance for a candidate
  */
 static float
-GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation)
+GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, HnswSupport * support)
 {
 	HnswElement hce = HnswPtrAccess(base, hc->element);
 	Datum		value = HnswGetValue(base, hce);
 
-	return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, value));
+	return DatumGetFloat8(FunctionCall2Coll(support->procinfo, support->collation, q, value));
 }
 
 /*
  * Create a candidate for the entry point
  */
 HnswCandidate *
-HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
+HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, HnswSupport * support, bool loadVec)
 {
 	HnswCandidate *hc = palloc(sizeof(HnswCandidate));
 
 	HnswPtrStore(base, hc->element, entryPoint);
 	if (index == NULL)
-		hc->distance = GetCandidateDistance(base, hc, q, procinfo, collation);
+		hc->distance = GetCandidateDistance(base, hc, q, support);
 	else
-		HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec);
+		HnswLoadElement(entryPoint, &hc->distance, &q, index, support, loadVec);
 	return hc;
 }
 
@@ -715,7 +732,7 @@ CountElement(char *base, HnswElement skipElement, HnswCandidate * hc)
  * Algorithm 2 from paper
  */
 List *
-HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
+HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement)
 {
 	List	   *w = NIL;
 	pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
@@ -731,7 +748,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 	/* Create local memory for neighborhood if needed */
 	if (index == NULL)
 	{
-		neighborhoodSize = offsetof(HnswNeighborArray, items) + sizeof(HnswCandidate) * HnswGetLayerM(m, lc);
+		neighborhoodSize = HNSW_NEIGHBOR_ARRAY_SIZE(HnswGetLayerM(m, lc));
 		neighborhoodData = palloc(neighborhoodSize);
 	}
 
@@ -797,9 +814,9 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 				f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
 
 				if (index == NULL)
-					eDistance = GetCandidateDistance(base, e, q, procinfo, collation);
+					eDistance = GetCandidateDistance(base, e, q, support);
 				else
-					HnswLoadElement(eElement, &eDistance, &q, index, procinfo, collation, inserting);
+					HnswLoadElement(eElement, &eDistance, &q, index, support, inserting);
 
 				Assert(!eElement->deleted);
 
@@ -907,49 +924,19 @@ CompareCandidateDistancesOffset(const void *a, const void *b)
  * Calculate the distance between elements
  */
 static float
-HnswGetDistance(char *base, HnswElement a, HnswElement b, int lc, FmgrInfo *procinfo, Oid collation)
+HnswGetDistance(char *base, HnswElement a, HnswElement b, HnswSupport * support)
 {
-	Datum		aValue;
+	Datum		aValue = HnswGetValue(base, a);
-	Datum		bValue;
+	Datum		bValue = HnswGetValue(base, b);
 
-	/* Look for cached distance */
+	return DatumGetFloat8(FunctionCall2Coll(support->procinfo, support->collation, aValue, bValue));
-	if (!HnswPtrIsNull(base, a->neighbors))
-	{
-		HnswNeighborArray *neighbors = HnswGetNeighbors(base, a, lc);
-
-		for (int i = 0; i < neighbors->length; i++)
-		{
-			HnswElement element = HnswPtrAccess(base, neighbors->items[i].element);
-
-			if (element == b)
-				return neighbors->items[i].distance;
-		}
-	}
-
-	if (!HnswPtrIsNull(base, b->neighbors))
-	{
-		HnswNeighborArray *neighbors = HnswGetNeighbors(base, b, lc);
-
-		for (int i = 0; i < neighbors->length; i++)
-		{
-			HnswElement element = HnswPtrAccess(base, neighbors->items[i].element);
-
-			if (element == a)
-				return neighbors->items[i].distance;
-		}
-	}
-
-	aValue = HnswGetValue(base, a);
-	bValue = HnswGetValue(base, b);
-
-	return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, aValue, bValue));
 }
 
 /*
  * Check if an element is closer to q than any element from R
  */
 static bool
-CheckElementCloser(char *base, HnswCandidate * e, List *r, int lc, FmgrInfo *procinfo, Oid collation)
+CheckElementCloser(char *base, HnswCandidate * e, List *r, HnswSupport * support)
 {
 	HnswElement eElement = HnswPtrAccess(base, e->element);
 	ListCell   *lc2;
@@ -958,7 +945,7 @@ CheckElementCloser(char *base, HnswCandidate * e, List *r, int lc, FmgrInfo *pro
 	{
 		HnswCandidate *ri = lfirst(lc2);
 		HnswElement riElement = HnswPtrAccess(base, ri->element);
-		float		distance = HnswGetDistance(base, eElement, riElement, lc, procinfo, collation);
+		float		distance = HnswGetDistance(base, eElement, riElement, support);
 
 		if (distance <= e->distance)
 			return false;
@@ -971,7 +958,7 @@ CheckElementCloser(char *base, HnswCandidate * e, List *r, int lc, FmgrInfo *pro
  * Algorithm 4 from paper
  */
 static List *
-SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid collation, HnswElement e2, HnswCandidate * newCandidate, HnswCandidate * *pruned, bool sortCandidates)
+SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, HnswElement e2, HnswCandidate * newCandidate, HnswCandidate * *pruned, bool sortCandidates)
 {
 	List	   *r = NIL;
 	List	   *w = list_copy(c);
@@ -1004,16 +991,20 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 
 		/* Use previous state of r and wd to skip work when possible */
 		if (mustCalculate)
-			e->closer = CheckElementCloser(base, e, r, lc, procinfo, collation);
+			e->closer = CheckElementCloser(base, e, r, support);
 		else if (list_length(added) > 0)
 		{
+			/* Keep Valgrind happy for in-memory, parallel builds */
+			if (base != NULL)
+				VALGRIND_MAKE_MEM_DEFINED(&e->closer, 1);
+
 			/*
 			 * If the current candidate was closer, we only need to compare it
 			 * with the other candidates that we have added.
 			 */
 			if (e->closer)
 			{
-				e->closer = CheckElementCloser(base, e, added, lc, procinfo, collation);
+				e->closer = CheckElementCloser(base, e, added, support);
 
 				if (!e->closer)
 					removedAny = true;
@@ -1026,7 +1017,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 				 */
 				if (removedAny)
 				{
-					e->closer = CheckElementCloser(base, e, r, lc, procinfo, collation);
+					e->closer = CheckElementCloser(base, e, r, support);
 					if (e->closer)
 						added = lappend(added, e);
 				}
@@ -1034,11 +1025,15 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 		}
 		else if (e == newCandidate)
 		{
-			e->closer = CheckElementCloser(base, e, r, lc, procinfo, collation);
+			e->closer = CheckElementCloser(base, e, r, support);
 			if (e->closer)
 				added = lappend(added, e);
 		}
 
+		/* Keep Valgrind happy for in-memory, parallel builds */
+		if (base != NULL)
+			VALGRIND_MAKE_MEM_DEFINED(&e->closer, 1);
+
 		if (e->closer)
 			r = lappend(r, e);
 		else
@@ -1081,7 +1076,7 @@ AddConnections(char *base, HnswElement element, List *neighbors, int lc)
  * Update connections
  */
 void
-HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation)
+HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, HnswSupport * support)
 {
 	HnswElement hce = HnswPtrAccess(base, hc->element);
 	HnswNeighborArray *currentNeighbors = HnswGetNeighbors(base, hce, lc);
@@ -1114,9 +1109,9 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
 				HnswElement hc3Element = HnswPtrAccess(base, hc3->element);
 
 				if (HnswPtrIsNull(base, hc3Element->value))
-					HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true);
+					HnswLoadElement(hc3Element, &hc3->distance, &q, index, support, true);
 				else
-					hc3->distance = GetCandidateDistance(base, hc3, q, procinfo, collation);
+					hc3->distance = GetCandidateDistance(base, hc3, q, support);
 
 				/* Prune element if being deleted */
 				if (hc3Element->heaptidsLength == 0)
@@ -1136,7 +1131,7 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
 				c = lappend(c, &currentNeighbors->items[i]);
 			c = lappend(c, &hc2);
 
-			SelectNeighbors(base, c, lm, lc, procinfo, collation, hce, &hc2, &pruned, true);
+			SelectNeighbors(base, c, lm, lc, support, hce, &hc2, &pruned, true);
 
 			/* Should not happen */
 			if (pruned == NULL)
@@ -1210,7 +1205,7 @@ PrecomputeHash(char *base, HnswElement element)
  * Algorithm 1 from paper
  */
 void
-HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing)
+HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing)
 {
 	List	   *ep;
 	List	   *w;
@@ -1230,13 +1225,13 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		return;
 
 	/* Get entry point and level */
-	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, true));
+	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, support, true));
 	entryLevel = entryPoint->level;
 
 	/* 1st phase: greedy search to insert level */
 	for (int lc = entryLevel; lc >= level + 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
+		w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, true, skipElement);
 		ep = w;
 	}
 
@@ -1254,7 +1249,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		List	   *neighbors;
 		List	   *lw;
 
-		w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
+		w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, support, m, true, skipElement);
 
 		/* Elements being deleted or skipped can help with search */
 		/* but should be removed before selecting neighbors */
@@ -1268,7 +1263,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		 * sortCandidates to true for in-memory builds to enable closer
 		 * caching, but there does not seem to be a difference in performance.
 		 */
-		neighbors = SelectNeighbors(base, lw, lm, lc, procinfo, collation, element, NULL, NULL, false);
+		neighbors = SelectNeighbors(base, lw, lm, lc, support, element, NULL, NULL, false);
 
 		AddConnections(base, element, neighbors, lc);
 

src/hnswvacuum.c

@@ -60,8 +60,7 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 		/* Iterate over nodes */
 		for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
 		{
-			ItemId		itemid = PageGetItemId(page, offno);
+			HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
-			HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
 			int			idx = 0;
 			bool		itemUpdated = false;
 
@@ -92,15 +91,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 				if (itemUpdated)
 				{
-					Size		etupSize = ItemIdGetLength(itemid);
-
 					/* Mark rest as invalid */
 					for (int i = idx; i < HNSW_HEAPTIDS; i++)
 						ItemPointerSetInvalid(&etup->heaptids[i]);
 
-					if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
-						elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
-
 					updated = true;
 				}
 			}
@@ -195,8 +189,6 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
 	GenericXLogState *state;
 	int			m = vacuumstate->m;
 	int			efConstruction = vacuumstate->efConstruction;
-	FmgrInfo   *procinfo = vacuumstate->procinfo;
-	Oid			collation = vacuumstate->collation;
 	BufferAccessStrategy bas = vacuumstate->bas;
 	HnswNeighborTuple ntup = vacuumstate->ntup;
 	Size		ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
@@ -211,7 +203,10 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
 	element->heaptidsLength = 0;
 
 	/* Find neighbors for element, skipping itself */
-	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
+	HnswFindElementNeighbors(base, element, entryPoint, index, &vacuumstate->support, m, efConstruction, true);
+
+	/* Zero memory for each element */
+	MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
 
 	/* Update neighbor tuple */
 	/* Do this before getting page to minimize locking */
@@ -232,7 +227,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
 	UnlockReleaseBuffer(buf);
 
 	/* Update neighbors */
-	HnswUpdateNeighborsOnDisk(index, procinfo, collation, element, m, true, false);
+	HnswUpdateNeighborsOnDisk(index, &vacuumstate->support, element, m, true, false);
 }
 
 /*
@@ -259,7 +254,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
 		LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
 
 		/* Load element */
-		HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
+		HnswLoadElement(highestPoint, NULL, NULL, index, &vacuumstate->support, true);
 
 		/* Repair if needed */
 		if (NeedsUpdated(vacuumstate, highestPoint))
@@ -297,7 +292,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
 			 * is outdated, this can remove connections at higher levels in
 			 * the graph until they are repaired, but this should be fine.
 			 */
-			HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
+			HnswLoadElement(entryPoint, NULL, NULL, index, &vacuumstate->support, true);
 
 			if (NeedsUpdated(vacuumstate, entryPoint))
 			{
@@ -479,11 +474,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 		/* Update element and neighbors together */
 		for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
 		{
-			ItemId		itemid = PageGetItemId(page, offno);
+			HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
-			HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
 			HnswNeighborTuple ntup;
-			Size		etupSize;
-			Size		ntupSize;
 			Buffer		nbuf;
 			Page		npage;
 			BlockNumber neighborPage;
@@ -507,10 +499,6 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 			if (ItemPointerIsValid(&etup->heaptids[0]))
 				continue;
 
-			/* Calculate sizes */
-			etupSize = ItemIdGetLength(itemid);
-			ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(etup->level, vacuumstate->m);
-
 			/* Get neighbor page */
 			neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
 			neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
@@ -537,13 +525,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 			for (int i = 0; i < ntup->count; i++)
 				ItemPointerSetInvalid(&ntup->indextids[i]);
 
-			/* Overwrite element tuple */
+			/*
-			if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
+			 * We modified the tuples in place, no need to call
-				elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
+			 * PageIndexTupleOverwrite
-
+			 */
-			/* Overwrite neighbor tuple */
-			if (!PageIndexTupleOverwrite(npage, neighborOffno, (Item) ntup, ntupSize))
-				elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
 
 			/* Commit */
 			GenericXLogFinish(state);
@@ -586,9 +571,9 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 	vacuumstate->callback_state = callback_state;
 	vacuumstate->efConstruction = HnswGetEfConstruction(index);
 	vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
-	vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+	vacuumstate->support.procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
-	vacuumstate->collation = index->rd_indcollation[0];
+	vacuumstate->support.collation = index->rd_indcollation[0];
-	vacuumstate->ntup = palloc0(BLCKSZ);
+	vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
 	vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
 												"Hnsw vacuum temporary context",
 												ALLOCSET_DEFAULT_SIZES);

src/ivfbuild.c

@@ -29,8 +29,6 @@
 #define CALLBACK_ITEM_POINTER HeapTuple hup
 #endif
 
-#define UpdateProgress(index, val) pgstat_progress_update_param(index, val)
-
 #if PG_VERSION_NUM >= 140000
 #include "utils/backend_status.h"
 #include "utils/wait_event.h"
@@ -262,9 +260,9 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 	TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsMinimalTuple);
 	TupleDesc	tupdesc = RelationGetDescr(index);
 
-	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
+	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
 
-	UpdateProgress(PROGRESS_CREATEIDX_TUPLES_TOTAL, buildstate->indtuples);
+	pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_TOTAL, buildstate->indtuples);
 
 	GetNextTuple(buildstate->sortstate, tupdesc, slot, &itup, &list);
 
@@ -300,7 +298,7 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 
 			pfree(itup);
 
-			UpdateProgress(PROGRESS_CREATEIDX_TUPLES_DONE, ++inserted);
+			pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, ++inserted);
 
 			GetNextTuple(buildstate->sortstate, tupdesc, slot, &itup, &list);
 		}
@@ -400,7 +398,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
 {
 	int			numSamples;
 
-	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
+	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
 
 	/* Target 50 samples per list, with at least 10000 samples */
 	/* The number of samples has a large effect on index build time */
@@ -475,7 +473,7 @@ CreateListPages(Relation index, VectorArray centers, int dimensions,
 	IvfflatList list;
 
 	listSize = MAXALIGN(IVFFLAT_LIST_SIZE(dimensions));
-	list = palloc(listSize);
+	list = palloc0(listSize);
 
 	buf = IvfflatNewBuffer(index, forkNum);
 	IvfflatInitRegisterPage(index, &buf, &page, &state);
@@ -921,7 +919,7 @@ AssignTuples(IvfflatBuildState * buildstate)
 	Oid			sortCollations[] = {InvalidOid};
 	bool		nullsFirstFlags[] = {false};
 
-	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN);
+	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN);
 
 	/* Calculate parallel workers */
 	if (buildstate->heap != NULL)

src/ivfscan.c

@@ -310,6 +310,7 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
 		ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
 
 		scan->xs_heaptid = *heaptid;
+		scan->xs_recheck = false;
 		scan->xs_recheckorderby = false;
 		return true;
 	}

src/vector.c

@@ -10,7 +10,6 @@
 #include "lib/stringinfo.h"
 #include "libpq/pqformat.h"
 #include "port.h"				/* for strtof() */
-#include "port/pg_bitutils.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
 #include "utils/float.h"
@@ -867,9 +866,10 @@ vector_mul(PG_FUNCTION_ARGS)
 int
 vector_cmp_internal(Vector * a, Vector * b)
 {
-	CheckDims(a, b);
+	int			dim = Min(a->dim, b->dim);
 
-	for (int i = 0; i < a->dim; i++)
+	/* Check values before dimensions to be consistent with Postgres arrays */
+	for (int i = 0; i < dim; i++)
 	{
 		if (a->x[i] < b->x[i])
 			return -1;
@@ -877,6 +877,13 @@ vector_cmp_internal(Vector * a, Vector * b)
 		if (a->x[i] > b->x[i])
 			return 1;
 	}
+
+	if (a->dim < b->dim)
+		return -1;
+
+	if (a->dim > b->dim)
+		return 1;
+
 	return 0;
 }
 
@@ -890,6 +897,9 @@ vector_lt(PG_FUNCTION_ARGS)
 	Vector	   *a = PG_GETARG_VECTOR_P(0);
 	Vector	   *b = PG_GETARG_VECTOR_P(1);
 
+	/* TODO Remove in 0.7.0 */
+	CheckDims(a, b);
+
 	PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
 }
 
@@ -903,6 +913,9 @@ vector_le(PG_FUNCTION_ARGS)
 	Vector	   *a = PG_GETARG_VECTOR_P(0);
 	Vector	   *b = PG_GETARG_VECTOR_P(1);
 
+	/* TODO Remove in 0.7.0 */
+	CheckDims(a, b);
+
 	PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
 }
 
@@ -916,6 +929,9 @@ vector_eq(PG_FUNCTION_ARGS)
 	Vector	   *a = PG_GETARG_VECTOR_P(0);
 	Vector	   *b = PG_GETARG_VECTOR_P(1);
 
+	/* TODO Remove in 0.7.0 */
+	CheckDims(a, b);
+
 	PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
 }
 
@@ -929,6 +945,9 @@ vector_ne(PG_FUNCTION_ARGS)
 	Vector	   *a = PG_GETARG_VECTOR_P(0);
 	Vector	   *b = PG_GETARG_VECTOR_P(1);
 
+	/* TODO Remove in 0.7.0 */
+	CheckDims(a, b);
+
 	PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
 }
 
@@ -942,6 +961,9 @@ vector_ge(PG_FUNCTION_ARGS)
 	Vector	   *a = PG_GETARG_VECTOR_P(0);
 	Vector	   *b = PG_GETARG_VECTOR_P(1);
 
+	/* TODO Remove in 0.7.0 */
+	CheckDims(a, b);
+
 	PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
 }
 
@@ -955,6 +977,9 @@ vector_gt(PG_FUNCTION_ARGS)
 	Vector	   *a = PG_GETARG_VECTOR_P(0);
 	Vector	   *b = PG_GETARG_VECTOR_P(1);
 
+	/* TODO Remove in 0.7.0 */
+	CheckDims(a, b);
+
 	PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
 }
 
@@ -1135,43 +1160,3 @@ vector_avg(PG_FUNCTION_ARGS)
 
 	PG_RETURN_POINTER(result);
 }
-
-/*
- * Ensure same number of bytes
- */
-static inline void
-CheckByteLengths(uint32 aLen, uint32 bLen)
-{
-	if (aLen != bLen)
-		ereport(ERROR,
-				(errcode(ERRCODE_DATA_EXCEPTION),
-				 errmsg("different byte lengths %u and %u", aLen, bLen)));
-}
-
-/*
- * Get the Hamming distance between two binary strings
- */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
-Datum
-hamming_distance(PG_FUNCTION_ARGS)
-{
-	bytea	   *a = PG_GETARG_BYTEA_PP(0);
-	bytea	   *b = PG_GETARG_BYTEA_PP(1);
-	char	   *ax = VARDATA_ANY(a);
-	char	   *bx = VARDATA_ANY(b);
-	uint32		aLen = VARSIZE_ANY_EXHDR(a);
-	uint32		bLen = VARSIZE_ANY_EXHDR(b);
-	uint64		distance = 0;
-
-	CheckByteLengths(aLen, bLen);
-
-	for (uint32 i = 0; i < aLen; i++)
-	{
-		unsigned char diff = (unsigned char) (ax[i] ^ bx[i]);
-
-		distance += pg_number_of_ones[diff];
-	}
-
-	/* TODO Decide on return type */
-	PG_RETURN_FLOAT8((double) distance);
-}

test/expected/functions.out

@@ -24,6 +24,56 @@ SELECT '[1e37]'::vector * '[1e37]';
 ERROR:  value out of range: overflow
 SELECT '[1e-37]'::vector * '[1e-37]';
 ERROR:  value out of range: underflow
+SELECT '[1,2,3]'::vector = '[1,2,3]';
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT '[1,2,3]'::vector = '[1,2]';
+ERROR:  different vector dimensions 3 and 2
+SELECT vector_cmp('[1,2,3]', '[1,2,3]');
+ vector_cmp 
+------------
+          0
+(1 row)
+
+SELECT vector_cmp('[1,2,3]', '[0,0,0]');
+ vector_cmp 
+------------
+          1
+(1 row)
+
+SELECT vector_cmp('[0,0,0]', '[1,2,3]');
+ vector_cmp 
+------------
+         -1
+(1 row)
+
+SELECT vector_cmp('[1,2]', '[1,2,3]');
+ vector_cmp 
+------------
+         -1
+(1 row)
+
+SELECT vector_cmp('[1,2,3]', '[1,2]');
+ vector_cmp 
+------------
+          1
+(1 row)
+
+SELECT vector_cmp('[1,2]', '[2,3,4]');
+ vector_cmp 
+------------
+         -1
+(1 row)
+
+SELECT vector_cmp('[2,3]', '[1,2,3]');
+ vector_cmp 
+------------
+          1
+(1 row)
+
 SELECT vector_dims('[1,2,3]');
  vector_dims 
 -------------
@@ -158,32 +208,6 @@ SELECT l1_distance('[3e38]', '[-3e38]');
     Infinity
 (1 row)
 
-SELECT hamming_distance('\xFFFF', '\xFFFF');
- hamming_distance 
-------------------
-                0
-(1 row)
-
-SELECT hamming_distance('\xFFFF', '\xFFFE');
- hamming_distance 
-------------------
-                1
-(1 row)
-
-SELECT hamming_distance('\xFFFF', '\xFFFC');
- hamming_distance 
-------------------
-                2
-(1 row)
-
-SELECT hamming_distance('\xFFFF', '\x0000');
- hamming_distance 
-------------------
-               16
-(1 row)
-
-SELECT hamming_distance('\xFFFF', '\x00');
-ERROR:  different byte lengths 2 and 1
 SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
     avg    
 -----------

test/sql/functions.sql

@@ -6,6 +6,17 @@ SELECT '[1,2,3]'::vector * '[4,5,6]';
 SELECT '[1e37]'::vector * '[1e37]';
 SELECT '[1e-37]'::vector * '[1e-37]';
 
+SELECT '[1,2,3]'::vector = '[1,2,3]';
+SELECT '[1,2,3]'::vector = '[1,2]';
+
+SELECT vector_cmp('[1,2,3]', '[1,2,3]');
+SELECT vector_cmp('[1,2,3]', '[0,0,0]');
+SELECT vector_cmp('[0,0,0]', '[1,2,3]');
+SELECT vector_cmp('[1,2]', '[1,2,3]');
+SELECT vector_cmp('[1,2,3]', '[1,2]');
+SELECT vector_cmp('[1,2]', '[2,3,4]');
+SELECT vector_cmp('[2,3]', '[1,2,3]');
+
 SELECT vector_dims('[1,2,3]');
 
 SELECT round(vector_norm('[1,1]')::numeric, 5);
@@ -37,12 +48,6 @@ SELECT l1_distance('[0,0]', '[0,1]');
 SELECT l1_distance('[1,2]', '[3]');
 SELECT l1_distance('[3e38]', '[-3e38]');
 
-SELECT hamming_distance('\xFFFF', '\xFFFF');
-SELECT hamming_distance('\xFFFF', '\xFFFE');
-SELECT hamming_distance('\xFFFF', '\xFFFC');
-SELECT hamming_distance('\xFFFF', '\x0000');
-SELECT hamming_distance('\xFFFF', '\x00');
-
 SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
 SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
 SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat and hnsw access methods'
-default_version = '0.5.1'
+default_version = '0.6.0'
 module_pathname = '$libdir/vector'
 relocatable = true