Added strict ordering [skip ci]

Previously, the cost estimation formula for a HNSW index scan utilized
a methodology that only factored in the entry level for an HNSW scan
and the "m" index parameter, which reflects the number of tuples (or
vectors) to scan at each step of a HNSW graph traversal. While this
would bias the PostgreSQL query planner to choose an HNSW index scan
over other available paths, this could lead to potential suboptimal
index selection, for example, choosing to use a HNSW index instead of
an available B-tree index that has better selectivity.

The number of tuples scanned during HNSW graph traversal is principally
influenced by these factors:

 * The number of tuples stored in the index
 * `m` - the number of tuples that are scanned in each step of the graph
   traversal
 * `hnsw.ef_search` - which influences the total number of steps it
   takes for the scan to converge on the approximated nearest neighbors

Through testing different source models for vectors, we also observed
that the correlation of vectors in mdoels would impact this convergence.
For this first iteration, we've opted to hardcode a constant scaling
factor and set it to `0.55`, though a future commit may turn this into
a configurable parameter.

The high-level formula for estimating the cost of a HNSW index scan is
as such:

```
(entryLevel * m) + (layer0TuplesMax * layer0Selectivity)
```

where

- `(entryLevel * m)` is the lower bound of tuples to scan, as it
accounts for the graph traversal to layer 0 (L0). (L1 and above has an ef=1)
- `layer0TuplesMax` is an estimate of the maximum number of tuples to
scan at L0. This accounts for tuples that may end up being discarded due
to them already being visited. Testing shows that the number of steps
until converge is similar to the value of `hnsw.ef_search`, thus we can
estimate tuples max at `hnsw.ef_search * m * 2`
- `layer0Selectivity` - estimates the percentage of tuples that will
actually be scanned during the index traversal, multipled by the scaling
factor

In addition to the `m` build parameter and `hsnw.ef_search`, costs
estimates can be influenced by standard PostgreSQL costing parameters,
though adjusting those (e.g. `random_page_cost`) should be done with
care.

Co-authored-by: @ankane

.github/workflows/build.yml

@@ -8,17 +8,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # - postgres: 18
+          #   os: ubuntu-24.04
           - postgres: 17
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - postgres: 16
             os: ubuntu-22.04
           - postgres: 15
             os: ubuntu-22.04
           - postgres: 14
-            os: ubuntu-22.04
-          - postgres: 13
             os: ubuntu-20.04
-          - postgres: 12
+          - postgres: 13
             os: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4

CHANGELOG.md

@@ -1,3 +1,26 @@
+## 0.8.0 (unreleased)
+
+- Added support for iterative index scans
+- Added casts for arrays to `sparsevec`
+- Improved cost estimation
+- Reduced memory usage for HNSW index scans
+- Dropped support for Postgres 12
+
+## 0.7.4 (2024-08-05)
+
+- Fixed locking for parallel HNSW index builds
+- Fixed compilation error with GCC 14 on i386 when SSE2 is not enabled
+
+## 0.7.3 (2024-07-22)
+
+- Fixed `failed to add index item` error with `sparsevec`
+- Fixed compilation error with FreeBSD ARM
+- Fixed compilation warning with MSVC and Postgres 16
+
+## 0.7.2 (2024-06-11)
+
+- Fixed initialization fork for indexes on unlogged tables
+
 ## 0.7.1 (2024-06-03)
 
 - Improved performance of on-disk HNSW index builds

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.7.1",
+	"version": "0.7.4",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.7.1",
+			"version": "0.7.4",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,8 +1,9 @@
 EXTENSION = vector
-EXTVERSION = 0.7.1
+EXTVERSION = 0.7.4
 
 MODULE_big = vector
-DATA = $(wildcard sql/*--*.sql)
+DATA = $(wildcard sql/*--*--*.sql)
+DATA_built = sql/$(EXTENSION)--$(EXTVERSION).sql
 OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
 HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
 
@@ -42,8 +43,6 @@ all: sql/$(EXTENSION)--$(EXTVERSION).sql
 sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql
 	cp $< $@
 
-EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql
-
 PG_CONFIG ?= pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
@@ -53,7 +52,7 @@ ifeq ($(PROVE),)
 	PROVE = prove
 endif
 
-# for Postgres 15
+# for Postgres < 15
 PROVE_FLAGS += -I ./test/perl
 
 prove_installcheck:

Makefile.win

@@ -1,6 +1,7 @@
 EXTENSION = vector
-EXTVERSION = 0.7.1
+EXTVERSION = 0.7.4
 
+DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
 OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
 HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
 
@@ -19,11 +20,6 @@ PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
 # https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
 # PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
 
-all: sql\$(EXTENSION)--$(EXTVERSION).sql
-
-sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
-	copy sql\$(EXTENSION).sql $@
-
 # TODO use pg_config
 !ifndef PGROOT
 !error PGROOT is not set
@@ -43,15 +39,18 @@ SHLIB = $(EXTENSION).dll
 
 LIBS = "$(LIBDIR)\postgres.lib"
 
+all: $(SHLIB) $(DATA_built)
+
 .c.obj:
 	$(CC) $(CFLAGS) /c $< /Fo$@
 
 $(SHLIB): $(OBJS)
 	$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
 
-all: $(SHLIB)
+sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
+	copy sql\$(EXTENSION).sql $@
 
-install:
+install: all
 	copy $(SHLIB) "$(PKGLIBDIR)"
 	copy $(EXTENSION).control "$(SHAREDIR)\extension"
 	copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
@@ -70,6 +69,6 @@ uninstall:
 
 clean:
 	del /f $(SHLIB) $(EXTENSION).lib $(EXTENSION).exp
+	del /f $(DATA_built)
 	del /f $(OBJS)
-	del /f sql\$(EXTENSION)--$(EXTVERSION).sql
 	del /f /s /q results regression.diffs regression.out tmp_check tmp_check_iso log output_iso

README.md

@@ -21,7 +21,7 @@ Compile and install the extension (supports Postgres 12+)
 
 ```sh
 cd /tmp
-git clone --branch v0.7.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -46,7 +46,7 @@ Then use `nmake` to build:
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\16"
 cd %TEMP%
-git clone --branch v0.7.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -106,7 +106,7 @@ Insert vectors
 INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
 ```
 
-Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py))
+Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py))
 
 ```sql
 COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -202,7 +202,7 @@ You can add an index to use approximate nearest neighbor search, which trades so
 
 Supported index types are:
 
-- [HNSW](#hnsw) - added in 0.5.0
+- [HNSW](#hnsw)
 - [IVFFlat](#ivfflat)
 
 ## HNSW
@@ -445,6 +445,63 @@ Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html
 CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
 ```
 
+## Streaming Queries [unreleased]
+
+*Added in 0.8.0*
+
+With approximate indexes, you can end up with less results than expected due to filtering conditions in the query.
+
+Starting with 0.8.0, you can enable streaming queries. If too few results from the initial index scan match the query filters, it will resume scanning until enough results are found. This can significantly improve recall (at the cost of speed).
+
+```tsql
+SET hnsw.streaming = on;
+-- or
+SET ivfflat.streaming = on;
+```
+
+### Streaming Options
+
+Since scanning a large portion of the index is expensive, there are options to control when the scan ends.
+
+#### HNSW
+
+Specify the max number of additional tuples visited
+
+```sql
+SET hnsw.ef_stream = 10000;
+```
+
+The scan will also end if reaches `work_mem`, at which point a notice is shown
+
+```text
+NOTICE:  hnsw index scan exceeded work_mem after 50000 tuples
+HINT:  Increase work_mem to scan more tuples.
+```
+
+Adjust this with:
+
+```sql
+SET work_mem = '8MB';
+```
+
+#### IVFFlat
+
+Specify the max number of probes
+
+```sql
+SET ivfflat.max_probes = 100;
+```
+
+### Streaming Order
+
+With streaming queries, it’s possible for rows to be slightly out of order by distance. For strict ordering, use:
+
+```sql
+WITH approx_order AS MATERIALIZED (
+    SELECT *, embedding <-> '[1,2,3]' AS distance FROM items WHERE ... ORDER BY distance LIMIT 5
+) SELECT * FROM approx_order ORDER BY distance;
+```
+
 ## Half-Precision Vectors
 
 *Added in 0.7.0*
@@ -473,7 +530,7 @@ SELECT * FROM items ORDER BY embedding::halfvec(3) <-> '[1,2,3]' LIMIT 5;
 
 ## Binary Vectors
 
-Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py))
+Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/imagehash/example.py))
 
 ```sql
 CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
@@ -551,7 +608,7 @@ SELECT id, content FROM items, plainto_tsquery('hello search') query
     WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
 ```
 
-You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search_rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py) to combine results.
+You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/cross_encoder.py) to combine results.
 
 ## Indexing Subvectors
 
@@ -597,7 +654,7 @@ Be sure to restart Postgres for changes to take effect.
 
 ### Loading
 
-Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py)).
+Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
 
 ```sql
 COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -687,7 +744,7 @@ Scale pgvector the same way you scale Postgres.
 
 Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
 
-Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
+Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
 
 ## Languages
 
@@ -1051,7 +1108,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.7.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build --pull --build-arg PG_MAJOR=16 -t myuser/pgvector .
 ```

sql/vector--0.7.1--0.7.2.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.2'" to load this file. \quit

sql/vector--0.7.2--0.7.3.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.3'" to load this file. \quit

sql/vector--0.7.3--0.7.4.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.4'" to load this file. \quit

sql/vector--0.7.4--0.8.0.sql

@@ -0,0 +1,26 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.0'" to load this file. \quit
+
+CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE CAST (integer[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
+
+CREATE CAST (real[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
+
+CREATE CAST (double precision[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
+
+CREATE CAST (numeric[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;

sql/vector.sql

@@ -782,6 +782,18 @@ CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparseve
 CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
 	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
 
+CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
+	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
 -- sparsevec casts
 
 CREATE CAST (sparsevec AS sparsevec)
@@ -799,6 +811,18 @@ CREATE CAST (sparsevec AS halfvec)
 CREATE CAST (halfvec AS sparsevec)
 	WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
 
+CREATE CAST (integer[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
+
+CREATE CAST (real[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
+
+CREATE CAST (double precision[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
+
+CREATE CAST (numeric[] AS sparsevec)
+	WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;
+
 -- sparsevec operators
 
 CREATE OPERATOR <-> (

src/bitutils.h

@@ -4,8 +4,8 @@
 #include "postgres.h"
 
 /* Check version in first header */
-#if PG_VERSION_NUM < 120000
-#error "Requires PostgreSQL 12+"
+#if PG_VERSION_NUM < 130000
+#error "Requires PostgreSQL 13+"
 #endif
 
 extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);

src/bitvec.c

@@ -3,6 +3,7 @@
 #include "bitutils.h"
 #include "bitvec.h"
 #include "utils/varbit.h"
+#include "vector.h"
 
 #if PG_VERSION_NUM >= 160000
 #include "varatt.h"
@@ -40,7 +41,7 @@ CheckDims(VarBit *a, VarBit *b)
 /*
  * Get the Hamming distance between two bit vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hamming_distance);
 Datum
 hamming_distance(PG_FUNCTION_ARGS)
 {
@@ -55,7 +56,7 @@ hamming_distance(PG_FUNCTION_ARGS)
 /*
  * Get the Jaccard distance between two bit vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(jaccard_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(jaccard_distance);
 Datum
 jaccard_distance(PG_FUNCTION_ARGS)
 {

src/halfvec.c

@@ -19,11 +19,6 @@
 #include "utils/numeric.h"
 #include "vector.h"
 
-#if PG_VERSION_NUM < 130000
-#define TYPALIGN_DOUBLE 'd'
-#define TYPALIGN_INT 'i'
-#endif
-
 #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
 #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
 
@@ -164,28 +159,10 @@ CheckStateArray(ArrayType *statearray, const char *caller)
 	return (float8 *) ARR_DATA_PTR(statearray);
 }
 
-#if PG_VERSION_NUM < 120003
-static pg_noinline void
-float_overflow_error(void)
-{
-	ereport(ERROR,
-			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
-			 errmsg("value out of range: overflow")));
-}
-
-static pg_noinline void
-float_underflow_error(void)
-{
-	ereport(ERROR,
-			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
-			 errmsg("value out of range: underflow")));
-}
-#endif
-
 /*
  * Convert textual representation to internal representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_in);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_in);
 Datum
 halfvec_in(PG_FUNCTION_ARGS)
 {
@@ -299,7 +276,7 @@ halfvec_in(PG_FUNCTION_ARGS)
 /*
  * Convert internal representation to textual representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_out);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_out);
 Datum
 halfvec_out(PG_FUNCTION_ARGS)
 {
@@ -345,7 +322,7 @@ halfvec_out(PG_FUNCTION_ARGS)
 /*
  * Convert type modifier
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_typmod_in);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_typmod_in);
 Datum
 halfvec_typmod_in(PG_FUNCTION_ARGS)
 {
@@ -376,7 +353,7 @@ halfvec_typmod_in(PG_FUNCTION_ARGS)
 /*
  * Convert external binary representation to internal representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_recv);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_recv);
 Datum
 halfvec_recv(PG_FUNCTION_ARGS)
 {
@@ -410,7 +387,7 @@ halfvec_recv(PG_FUNCTION_ARGS)
 /*
  * Convert internal representation to the external binary representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_send);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_send);
 Datum
 halfvec_send(PG_FUNCTION_ARGS)
 {
@@ -430,7 +407,7 @@ halfvec_send(PG_FUNCTION_ARGS)
  * Convert half vector to half vector
  * This is needed to check the type modifier
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec);
 Datum
 halfvec(PG_FUNCTION_ARGS)
 {
@@ -445,7 +422,7 @@ halfvec(PG_FUNCTION_ARGS)
 /*
  * Convert array to half vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_halfvec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_halfvec);
 Datum
 array_to_halfvec(PG_FUNCTION_ARGS)
 {
@@ -519,7 +496,7 @@ array_to_halfvec(PG_FUNCTION_ARGS)
 /*
  * Convert half vector to float4[]
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_float4);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_float4);
 Datum
 halfvec_to_float4(PG_FUNCTION_ARGS)
 {
@@ -543,7 +520,7 @@ halfvec_to_float4(PG_FUNCTION_ARGS)
 /*
  * Convert vector to half vec
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_halfvec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_halfvec);
 Datum
 vector_to_halfvec(PG_FUNCTION_ARGS)
 {
@@ -565,7 +542,7 @@ vector_to_halfvec(PG_FUNCTION_ARGS)
 /*
  * Get the L2 distance between half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_distance);
 Datum
 halfvec_l2_distance(PG_FUNCTION_ARGS)
 {
@@ -580,7 +557,7 @@ halfvec_l2_distance(PG_FUNCTION_ARGS)
 /*
  * Get the L2 squared distance between half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_squared_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_squared_distance);
 Datum
 halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
 {
@@ -595,7 +572,7 @@ halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
 /*
  * Get the inner product of two half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_inner_product);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_inner_product);
 Datum
 halfvec_inner_product(PG_FUNCTION_ARGS)
 {
@@ -610,7 +587,7 @@ halfvec_inner_product(PG_FUNCTION_ARGS)
 /*
  * Get the negative inner product of two half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_negative_inner_product);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_negative_inner_product);
 Datum
 halfvec_negative_inner_product(PG_FUNCTION_ARGS)
 {
@@ -625,7 +602,7 @@ halfvec_negative_inner_product(PG_FUNCTION_ARGS)
 /*
  * Get the cosine distance between two half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_cosine_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_cosine_distance);
 Datum
 halfvec_cosine_distance(PG_FUNCTION_ARGS)
 {
@@ -657,7 +634,7 @@ halfvec_cosine_distance(PG_FUNCTION_ARGS)
  * Currently uses angular distance since needs to satisfy triangle inequality
  * Assumes inputs are unit vectors (skips norm)
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_spherical_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_spherical_distance);
 Datum
 halfvec_spherical_distance(PG_FUNCTION_ARGS)
 {
@@ -681,7 +658,7 @@ halfvec_spherical_distance(PG_FUNCTION_ARGS)
 /*
  * Get the L1 distance between two half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l1_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l1_distance);
 Datum
 halfvec_l1_distance(PG_FUNCTION_ARGS)
 {
@@ -696,7 +673,7 @@ halfvec_l1_distance(PG_FUNCTION_ARGS)
 /*
  * Get the dimensions of a half vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_vector_dims);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_vector_dims);
 Datum
 halfvec_vector_dims(PG_FUNCTION_ARGS)
 {
@@ -708,7 +685,7 @@ halfvec_vector_dims(PG_FUNCTION_ARGS)
 /*
  * Get the L2 norm of a half vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_norm);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_norm);
 Datum
 halfvec_l2_norm(PG_FUNCTION_ARGS)
 {
@@ -730,7 +707,7 @@ halfvec_l2_norm(PG_FUNCTION_ARGS)
 /*
  * Normalize a half vector with the L2 norm
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_normalize);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_normalize);
 Datum
 halfvec_l2_normalize(PG_FUNCTION_ARGS)
 {
@@ -769,7 +746,7 @@ halfvec_l2_normalize(PG_FUNCTION_ARGS)
 /*
  * Add half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_add);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_add);
 Datum
 halfvec_add(PG_FUNCTION_ARGS)
 {
@@ -808,7 +785,7 @@ halfvec_add(PG_FUNCTION_ARGS)
 /*
  * Subtract half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_sub);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_sub);
 Datum
 halfvec_sub(PG_FUNCTION_ARGS)
 {
@@ -847,7 +824,7 @@ halfvec_sub(PG_FUNCTION_ARGS)
 /*
  * Multiply half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_mul);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_mul);
 Datum
 halfvec_mul(PG_FUNCTION_ARGS)
 {
@@ -889,7 +866,7 @@ halfvec_mul(PG_FUNCTION_ARGS)
 /*
  * Concatenate half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_concat);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_concat);
 Datum
 halfvec_concat(PG_FUNCTION_ARGS)
 {
@@ -913,7 +890,7 @@ halfvec_concat(PG_FUNCTION_ARGS)
 /*
  * Quantize a half vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_binary_quantize);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_binary_quantize);
 Datum
 halfvec_binary_quantize(PG_FUNCTION_ARGS)
 {
@@ -931,7 +908,7 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
 /*
  * Get a subvector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_subvector);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_subvector);
 Datum
 halfvec_subvector(PG_FUNCTION_ARGS)
 {
@@ -1005,7 +982,7 @@ halfvec_cmp_internal(HalfVector * a, HalfVector * b)
 /*
  * Less than
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_lt);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_lt);
 Datum
 halfvec_lt(PG_FUNCTION_ARGS)
 {
@@ -1018,7 +995,7 @@ halfvec_lt(PG_FUNCTION_ARGS)
 /*
  * Less than or equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_le);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_le);
 Datum
 halfvec_le(PG_FUNCTION_ARGS)
 {
@@ -1031,7 +1008,7 @@ halfvec_le(PG_FUNCTION_ARGS)
 /*
  * Equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_eq);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_eq);
 Datum
 halfvec_eq(PG_FUNCTION_ARGS)
 {
@@ -1044,7 +1021,7 @@ halfvec_eq(PG_FUNCTION_ARGS)
 /*
  * Not equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ne);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_ne);
 Datum
 halfvec_ne(PG_FUNCTION_ARGS)
 {
@@ -1057,7 +1034,7 @@ halfvec_ne(PG_FUNCTION_ARGS)
 /*
  * Greater than or equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ge);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_ge);
 Datum
 halfvec_ge(PG_FUNCTION_ARGS)
 {
@@ -1070,7 +1047,7 @@ halfvec_ge(PG_FUNCTION_ARGS)
 /*
  * Greater than
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_gt);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_gt);
 Datum
 halfvec_gt(PG_FUNCTION_ARGS)
 {
@@ -1083,7 +1060,7 @@ halfvec_gt(PG_FUNCTION_ARGS)
 /*
  * Compare half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_cmp);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_cmp);
 Datum
 halfvec_cmp(PG_FUNCTION_ARGS)
 {
@@ -1096,7 +1073,7 @@ halfvec_cmp(PG_FUNCTION_ARGS)
 /*
  * Accumulate half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_accum);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_accum);
 Datum
 halfvec_accum(PG_FUNCTION_ARGS)
 {
@@ -1157,7 +1134,7 @@ halfvec_accum(PG_FUNCTION_ARGS)
 /*
  * Average half vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_avg);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_avg);
 Datum
 halfvec_avg(PG_FUNCTION_ARGS)
 {
@@ -1191,7 +1168,7 @@ halfvec_avg(PG_FUNCTION_ARGS)
 /*
  * Convert sparse vector to half vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_halfvec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_halfvec);
 Datum
 sparsevec_to_halfvec(PG_FUNCTION_ARGS)
 {

src/halfvec.h

@@ -38,7 +38,7 @@
 /* F16C has better performance than _Float16 (on x86-64) */
 #if defined(__F16C__)
 #define F16C_SUPPORT
-#elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH)
+#elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH) && !defined(__FreeBSD__) && (!defined(__i386__) || defined(__SSE2__))
 #define FLT16_SUPPORT
 #endif
 

src/hnsw.c

@@ -9,14 +9,18 @@
 #include "commands/vacuum.h"
 #include "hnsw.h"
 #include "miscadmin.h"
+#include "utils/float.h"
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
+#include "utils/spccache.h"
 
 #if PG_VERSION_NUM < 150000
 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
 #endif
 
 int			hnsw_ef_search;
+int			hnsw_ef_stream;
+bool		hnsw_streaming;
 int			hnsw_lock_tranche_id;
 static relopt_kind hnsw_relopt_kind;
 
@@ -59,22 +63,25 @@ HnswInit(void)
 
 	hnsw_relopt_kind = add_reloption_kind();
 	add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",
-					  HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M
-#if PG_VERSION_NUM >= 130000
-					  ,AccessExclusiveLock
-#endif
-		);
+					  HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M, AccessExclusiveLock);
 	add_int_reloption(hnsw_relopt_kind, "ef_construction", "Size of the dynamic candidate list for construction",
-					  HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION
-#if PG_VERSION_NUM >= 130000
-					  ,AccessExclusiveLock
-#endif
-		);
+					  HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION, AccessExclusiveLock);
 
 	DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
 							"Valid range is 1..1000.", &hnsw_ef_search,
 							HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
 
+	/* TODO Figure out name */
+	DefineCustomBoolVariable("hnsw.streaming", "Use streaming mode",
+							 NULL, &hnsw_streaming,
+							 HNSW_DEFAULT_STREAMING, PGC_USERSET, 0, NULL, NULL, NULL);
+
+	/* TODO Figure out name */
+	/* TODO Use same value as ivfflat.max_probes for "all" */
+	DefineCustomIntVariable("hnsw.ef_stream", "Sets the max number of additional candidates to visit for streaming search",
+							"-1 means all", &hnsw_ef_stream,
+							HNSW_DEFAULT_EF_STREAM, HNSW_MIN_EF_STREAM, HNSW_MAX_EF_STREAM, PGC_USERSET, 0, NULL, NULL, NULL);
+
 	MarkGUCPrefixReserved("hnsw");
 }
 
@@ -95,6 +102,33 @@ hnswbuildphasename(int64 phasenum)
 	}
 }
 
+/*
+ * Estimate ef needed for iterative scans
+ */
+static int
+EstimateEf(PlannerInfo *root, IndexPath *path)
+{
+	double		selectivity = 1;
+	ListCell   *lc;
+
+	/* Cannot estimate without limit */
+	/* limit_tuples includes offset */
+	if (root->limit_tuples < 0)
+		return 0;
+
+	/* Get the selectivity of non-index conditions */
+	foreach(lc, path->indexinfo->indrestrictinfo)
+	{
+		RestrictInfo *rinfo = lfirst(lc);
+
+		/* Skip DEFAULT_INEQ_SEL since it may be a distance filter */
+		if (rinfo->norm_selec >= 0 && rinfo->norm_selec <= 1 && rinfo->norm_selec != (Selectivity) DEFAULT_INEQ_SEL)
+			selectivity *= rinfo->norm_selec;
+	}
+
+	return root->limit_tuples / Max(selectivity, 0.00001);
+}
+
 /*
  * Estimate the cost of an index scan
  */
@@ -106,14 +140,19 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 {
 	GenericCosts costs;
 	int			m;
+	int			ef;
 	int			entryLevel;
+	int			layer0TuplesMax;
+	double		layer0Selectivity;
+	double		scalingFactor = 0.55;
+	double		spc_seq_page_cost;
 	Relation	index;
 
 	/* Never use index without order */
 	if (path->indexorderbys == NULL)
 	{
-		*indexStartupCost = DBL_MAX;
-		*indexTotalCost = DBL_MAX;
+		*indexStartupCost = get_float8_infinity();
+		*indexTotalCost = get_float8_infinity();
 		*indexSelectivity = 0;
 		*indexCorrelation = 0;
 		*indexPages = 0;
@@ -126,15 +165,57 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	HnswGetMetaPageInfo(index, &m, NULL);
 	index_close(index, NoLock);
 
-	/* Approximate entry level */
-	entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
+	ef = hnsw_streaming ? Max(hnsw_ef_search, EstimateEf(root, path)) : hnsw_ef_search;
 
-	/* TODO Improve estimate of visited tuples (currently underestimates) */
-	/* Account for number of tuples (or entry level), m, and ef_search */
-	costs.numIndexTuples = (entryLevel + 2) * m;
+	/*
+	 * HNSW cost estimation follows a formula that accounts for the total
+	 * number of tuples indexed combined with the parameters that most
+	 * influence the duration of the index scan, namely: m - the number of
+	 * tuples that are scanned in each step of the HNSW graph traversal
+	 * ef_search - which influences the total number of steps taken at layer 0
+	 *
+	 * The source of the vector data can impact how many steps it takes to
+	 * converge on the set of vectors to return to the executor. Currently, we
+	 * use a hardcoded scaling factor (HNSWScanScalingFactor) to help
+	 * influence that, but this could later become a configurable parameter
+	 * based on the cost estimations.
+	 *
+	 * The tuple estimator formula is below:
+	 *
+	 * numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
+	 *
+	 * "entryLevel * m" represents the floor of tuples we need to scan to get
+	 * to layer 0 (L0).
+	 *
+	 * "layer0TuplesMax" is the estimated total number of tuples we'd scan at
+	 * L0 if we weren't discarding already visited tuples as part of the scan.
+	 *
+	 * "layer0Selectivity" estimates the percentage of tuples that are scanned
+	 * at L0, accounting for previously visited tuples, multiplied by the
+	 * "scalingFactor" (currently hardcoded).
+	 */
+	entryLevel = (int) (log(path->indexinfo->tuples + 1) * HnswGetMl(m));
+	layer0TuplesMax = HnswGetLayerM(m, 0) * ef;
+	layer0Selectivity = (scalingFactor * log(path->indexinfo->tuples + 1)) /
+		(log(m) * (1 + log(ef)));
+
+	costs.numIndexTuples = (entryLevel * m) +
+		(layer0TuplesMax * layer0Selectivity);
 
 	genericcostestimate(root, path, loop_count, &costs);
 
+	get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
+
+	/* Adjust cost if needed since TOAST not included in seq scan cost */
+	if (costs.numIndexPages > path->indexinfo->rel->pages)
+	{
+		/* Change all page cost from random to sequential */
+		costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+
+		/* Remove cost of extra pages */
+		costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
+	}
+
 	/* Use total cost since most work happens before first tuple is returned */
 	*indexStartupCost = costs.indexTotalCost;
 	*indexTotalCost = costs.indexTotalCost;
@@ -154,23 +235,10 @@ hnswoptions(Datum reloptions, bool validate)
 		{"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)},
 	};
 
-#if PG_VERSION_NUM >= 130000
 	return (bytea *) build_reloptions(reloptions, validate,
 									  hnsw_relopt_kind,
 									  sizeof(HnswOptions),
 									  tab, lengthof(tab));
-#else
-	relopt_value *options;
-	int			numoptions;
-	HnswOptions *rdopts;
-
-	options = parseRelOptions(reloptions, validate, hnsw_relopt_kind, &numoptions);
-	rdopts = allocateReloptStruct(sizeof(HnswOptions), options, numoptions);
-	fillRelOptions((void *) rdopts, sizeof(HnswOptions), options, numoptions,
-				   validate, tab, lengthof(tab));
-
-	return (bytea *) rdopts;
-#endif
 }
 
 /*
@@ -187,7 +255,7 @@ hnswvalidate(Oid opclassoid)
  *
  * See https://www.postgresql.org/docs/current/index-api.html
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(hnswhandler);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
 Datum
 hnswhandler(PG_FUNCTION_ARGS)
 {
@@ -195,9 +263,7 @@ hnswhandler(PG_FUNCTION_ARGS)
 
 	amroutine->amstrategies = 0;
 	amroutine->amsupport = 3;
-#if PG_VERSION_NUM >= 130000
 	amroutine->amoptsprocnum = 0;
-#endif
 	amroutine->amcanorder = false;
 	amroutine->amcanorderbyop = true;
 	amroutine->amcanbackward = false;	/* can change direction mid-scan */
@@ -210,17 +276,24 @@ hnswhandler(PG_FUNCTION_ARGS)
 	amroutine->amclusterable = false;
 	amroutine->ampredlocks = false;
 	amroutine->amcanparallel = false;
-	amroutine->amcaninclude = false;
-#if PG_VERSION_NUM >= 130000
-	amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
-	amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
+#if PG_VERSION_NUM >= 170000
+	amroutine->amcanbuildparallel = true;
 #endif
+	amroutine->amcaninclude = false;
+	amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
+#if PG_VERSION_NUM >= 160000
+	amroutine->amsummarizing = false;
+#endif
+	amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
 	amroutine->amkeytype = InvalidOid;
 
 	/* Interface functions */
 	amroutine->ambuild = hnswbuild;
 	amroutine->ambuildempty = hnswbuildempty;
 	amroutine->aminsert = hnswinsert;
+#if PG_VERSION_NUM >= 170000
+	amroutine->aminsertcleanup = NULL;
+#endif
 	amroutine->ambulkdelete = hnswbulkdelete;
 	amroutine->amvacuumcleanup = hnswvacuumcleanup;
 	amroutine->amcanreturn = NULL;

src/hnsw.h

@@ -12,6 +12,10 @@
 #include "utils/sampling.h"
 #include "vector.h"
 
+#ifdef HNSW_BENCH
+#include "portability/instr_time.h"
+#endif
+
 #define HNSW_MAX_DIM 2000
 #define HNSW_MAX_NNZ 1000
 
@@ -42,6 +46,10 @@
 #define HNSW_DEFAULT_EF_SEARCH	40
 #define HNSW_MIN_EF_SEARCH		1
 #define HNSW_MAX_EF_SEARCH		1000
+#define HNSW_DEFAULT_STREAMING	false
+#define HNSW_DEFAULT_EF_STREAM	-1
+#define HNSW_MIN_EF_STREAM		-1
+#define HNSW_MAX_EF_STREAM		INT_MAX
 
 /* Tuple types */
 #define HNSW_ELEMENT_TUPLE_TYPE  1
@@ -68,6 +76,21 @@
 #define HnswPageGetOpaque(page)	((HnswPageOpaque) PageGetSpecialPointer(page))
 #define HnswPageGetMeta(page)	((HnswMetaPageData *) PageGetContents(page))
 
+#ifdef HNSW_BENCH
+#define HnswBench(name, code) \
+	do { \
+		instr_time	start; \
+		instr_time	duration; \
+		INSTR_TIME_SET_CURRENT(start); \
+		(code); \
+		INSTR_TIME_SET_CURRENT(duration); \
+		INSTR_TIME_SUBTRACT(duration, start); \
+		elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
+	} while (0)
+#else
+#define HnswBench(name, code) (code)
+#endif
+
 #if PG_VERSION_NUM >= 150000
 #define RandomDouble() pg_prng_double(&pg_global_prng_state)
 #define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -76,11 +99,6 @@
 #define SeedRandom(seed) srandom(seed)
 #endif
 
-#if PG_VERSION_NUM < 130000
-#define list_delete_last(list) list_truncate(list, list_length(list) - 1)
-#define list_sort(list, cmp) ((list) = list_qsort(list, cmp))
-#endif
-
 #define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
 #define HnswIsNeighborTuple(tup) ((tup)->type == HNSW_NEIGHBOR_TUPLE_TYPE)
 
@@ -111,6 +129,8 @@
 
 /* Variables */
 extern int	hnsw_ef_search;
+extern int	hnsw_ef_stream;
+extern bool hnsw_streaming;
 extern int	hnsw_lock_tranche_id;
 
 typedef struct HnswElementData HnswElementData;
@@ -134,6 +154,7 @@ struct HnswElementData
 	uint8		heaptidsLength;
 	uint8		level;
 	uint8		deleted;
+	uint8		version;
 	uint32		hash;
 	HnswNeighborsPtr neighbors;
 	BlockNumber blkno;
@@ -160,11 +181,16 @@ struct HnswNeighborArray
 	HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
 };
 
-typedef struct HnswPairingHeapNode
+typedef struct HnswSearchCandidate
 {
-	pairingheap_node ph_node;
-	HnswCandidate *inner;
-}			HnswPairingHeapNode;
+	pairingheap_node c_node;
+	pairingheap_node w_node;
+	HnswElementPtr element;
+	float		distance;
+}			HnswSearchCandidate;
+
+#define HnswGetSearchCandidate(membername, ptr) pairingheap_container(HnswSearchCandidate, membername, ptr)
+#define HnswGetSearchCandidateConst(membername, ptr) pairingheap_const_container(HnswSearchCandidate, membername, ptr)
 
 /* HNSW index options */
 typedef struct HnswOptions
@@ -309,10 +335,10 @@ typedef struct HnswElementTupleData
 	uint8		type;
 	uint8		level;
 	uint8		deleted;
-	uint8		unused;
+	uint8		version;
 	ItemPointerData heaptids[HNSW_HEAPTIDS];
 	ItemPointerData neighbortid;
-	uint16		unused2;
+	uint16		unused;
 	Vector		data;
 }			HnswElementTupleData;
 
@@ -321,18 +347,31 @@ typedef HnswElementTupleData * HnswElementTuple;
 typedef struct HnswNeighborTupleData
 {
 	uint8		type;
-	uint8		unused;
+	uint8		version;
 	uint16		count;
 	ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
 }			HnswNeighborTupleData;
 
 typedef HnswNeighborTupleData * HnswNeighborTuple;
 
+typedef union
+{
+	struct pointerhash_hash *pointers;
+	struct offsethash_hash *offsets;
+	struct tidhash_hash *tids;
+}			visited_hash;
+
 typedef struct HnswScanOpaqueData
 {
 	const		HnswTypeInfo *typeInfo;
 	bool		first;
 	List	   *w;
+	visited_hash v;
+	pairingheap *discarded;
+	Datum		q;
+	int			m;
+	int64		tuples;
+	double		previousDistance;
 	MemoryContext tmpCtx;
 
 	/* Support functions */
@@ -378,14 +417,14 @@ bool		HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
 Buffer		HnswNewBuffer(Relation index, ForkNumber forkNum);
 void		HnswInitPage(Buffer buf, Page page);
 void		HnswInit(void);
-List	   *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
+List	   *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
 HnswElement HnswGetEntryPoint(Relation index);
 void		HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
 void	   *HnswAlloc(HnswAllocator * allocator, Size size);
 HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
 HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
 void		HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
-HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
+HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
 void		HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
 void		HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
 void		HnswAddHeapTid(HnswElement element, ItemPointer heaptid);

src/hnswbuild.c

@@ -60,12 +60,6 @@
 #include "pgstat.h"
 #endif
 
-#if PG_VERSION_NUM >= 130000
-#define CALLBACK_ITEM_POINTER ItemPointer tid
-#else
-#define CALLBACK_ITEM_POINTER HeapTuple hup
-#endif
-
 #if PG_VERSION_NUM >= 140000
 #include "utils/backend_status.h"
 #include "utils/wait_event.h"
@@ -75,10 +69,6 @@
 #define PARALLEL_KEY_HNSW_AREA			UINT64CONST(0xA000000000000002)
 #define PARALLEL_KEY_QUERY_TEXT			UINT64CONST(0xA000000000000003)
 
-#if PG_VERSION_NUM < 130000
-#define GENERATIONCHUNK_RAWSIZE (SIZEOF_SIZE_T + SIZEOF_VOID_P * 2)
-#endif
-
 /*
  * Create the metapage
  */
@@ -192,7 +182,9 @@ CreateGraphPages(HnswBuildState * buildstate)
 
 		/* Initial size check */
 		if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
-			elog(ERROR, "index tuple too large");
+			ereport(ERROR,
+					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+					 errmsg("index tuple too large")));
 
 		HnswSetElementTuple(base, etup, element);
 
@@ -379,7 +371,13 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
 	for (int lc = e->level; lc >= 0; lc--)
 	{
 		int			lm = HnswGetLayerM(m, lc);
-		HnswNeighborArray *neighbors = HnswGetNeighbors(base, e, lc);
+		Size		neighborsSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm);
+		HnswNeighborArray *neighbors = palloc(neighborsSize);
+
+		/* Copy neighbors to local memory */
+		LWLockAcquire(&e->lock, LW_SHARED);
+		memcpy(neighbors, HnswGetNeighbors(base, e, lc), neighborsSize);
+		LWLockRelease(&e->lock);
 
 		for (int i = 0; i < neighbors->length; i++)
 		{
@@ -389,7 +387,6 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
 			/* Keep scan-build happy on Mac x86-64 */
 			Assert(neighborElement);
 
-			/* Use element for lock instead of hc since hc can be replaced */
 			LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
 			HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
 			LWLockRelease(&neighborElement->lock);
@@ -578,17 +575,13 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
  * Callback for table_index_build_scan
  */
 static void
-BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
+BuildCallback(Relation index, ItemPointer tid, Datum *values,
 			  bool *isnull, bool tupleIsAlive, void *state)
 {
 	HnswBuildState *buildstate = (HnswBuildState *) state;
 	HnswGraph  *graph = buildstate->graph;
 	MemoryContext oldCtx;
 
-#if PG_VERSION_NUM < 130000
-	ItemPointer tid = &hup->t_self;
-#endif
-
 	/* Skip nulls */
 	if (isnull[0])
 		return;
@@ -651,11 +644,7 @@ HnswMemoryContextAlloc(Size size, void *state)
 	HnswBuildState *buildstate = (HnswBuildState *) state;
 	void	   *chunk = MemoryContextAlloc(buildstate->graphCtx, size);
 
-#if PG_VERSION_NUM >= 130000
 	buildstate->graphData.memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
-#else
-	buildstate->graphData.memoryUsed += MAXALIGN(size);
-#endif
 
 	return chunk;
 }
@@ -691,17 +680,25 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
 
 	/* Disallow varbit since require fixed dimensions */
 	if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
-		elog(ERROR, "type not supported for hnsw index");
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("type not supported for hnsw index")));
 
 	/* Require column to have dimensions to be indexed */
 	if (buildstate->dimensions < 0)
-		elog(ERROR, "column does not have dimensions");
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("column does not have dimensions")));
 
 	if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
-		elog(ERROR, "column cannot have more than %d dimensions for hnsw index", buildstate->typeInfo->maxDimensions);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("column cannot have more than %d dimensions for hnsw index", buildstate->typeInfo->maxDimensions)));
 
 	if (buildstate->efConstruction < 2 * buildstate->m)
-		elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("ef_construction must be greater than or equal to 2 * m")));
 
 	buildstate->reltuples = 0;
 	buildstate->indtuples = 0;
@@ -1121,8 +1118,8 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
 
 	BuildGraph(buildstate, forkNum);
 
-	if (RelationNeedsWAL(index))
-		log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocks(index), true);
+	if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
+		log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
 
 	FreeBuildState(buildstate);
 }

src/hnswinsert.c

@@ -36,14 +36,15 @@ GetInsertPage(Relation index)
  * Check for a free offset
  */
 static bool
-HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
+HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage, uint8 *tupleVersion)
 {
 	OffsetNumber offno;
 	OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
 
 	for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
 	{
-		HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
+		ItemId		eitemid = PageGetItemId(page, offno);
+		HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, eitemid);
 
 		/* Skip neighbor tuples */
 		if (!HnswIsElementTuple(etup))
@@ -54,7 +55,9 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
 			BlockNumber elementPage = BufferGetBlockNumber(buf);
 			BlockNumber neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
 			OffsetNumber neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
-			ItemId		itemid;
+			ItemId		nitemid;
+			Size		pageFree;
+			Size		npageFree;
 
 			if (!BlockNumberIsValid(*newInsertPage))
 				*newInsertPage = elementPage;
@@ -73,13 +76,29 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
 				*npage = BufferGetPage(*nbuf);
 			}
 
-			itemid = PageGetItemId(*npage, neighborOffno);
+			nitemid = PageGetItemId(*npage, neighborOffno);
 
-			/* Check for space on neighbor tuple page */
-			if (PageGetFreeSpace(*npage) + ItemIdGetLength(itemid) - sizeof(ItemIdData) >= ntupSize)
+			/* Ensure aligned for space check */
+			Assert(etupSize == MAXALIGN(etupSize));
+			Assert(ntupSize == MAXALIGN(ntupSize));
+
+			/*
+			 * Calculate free space individually since tuples are overwritten
+			 * individually (in separate calls to PageIndexTupleOverwrite)
+			 */
+			pageFree = ItemIdGetLength(eitemid) + PageGetExactFreeSpace(page);
+			npageFree = ItemIdGetLength(nitemid);
+			if (neighborPage != elementPage)
+				npageFree += PageGetExactFreeSpace(*npage);
+			else if (pageFree >= etupSize)
+				npageFree += pageFree - etupSize;
+
+			/* Check for space */
+			if (pageFree >= etupSize && npageFree >= ntupSize)
 			{
 				*freeOffno = offno;
 				*freeNeighborOffno = neighborOffno;
+				*tupleVersion = etup->version;
 				return true;
 			}
 			else if (*nbuf != buf)
@@ -135,6 +154,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 	OffsetNumber freeOffno = InvalidOffsetNumber;
 	OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
 	BlockNumber newInsertPage = InvalidBlockNumber;
+	uint8		tupleVersion;
 	char	   *base = NULL;
 
 	/* Calculate sizes */
@@ -184,7 +204,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 		}
 
 		/* Next, try space from a deleted element */
-		if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
+		if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage, &tupleVersion))
 		{
 			if (nbuf != buf)
 			{
@@ -194,6 +214,10 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 					npage = GenericXLogRegisterBuffer(state, nbuf, 0);
 			}
 
+			/* Set tuple version */
+			etup->version = tupleVersion;
+			ntup->version = tupleVersion;
+
 			break;
 		}
 
@@ -361,8 +385,12 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
 			HnswElement neighborElement = HnswPtrAccess(base, hc->element);
 			OffsetNumber offno = neighborElement->neighborOffno;
 
-			/* Get latest neighbors since they may have changed */
-			/* Do not lock yet since selecting neighbors can take time */
+			/*
+			 * Get latest neighbors since they may have changed. Do not lock
+			 * yet since selecting neighbors can take time. Could use
+			 * optimistic locking to retry if another update occurs before
+			 * getting exclusive lock.
+			 */
 			HnswLoadNeighbors(neighborElement, index, m);
 
 			/*

src/hnswscan.c

@@ -1,5 +1,7 @@
 #include "postgres.h"
 
+#include <float.h>
+
 #include "access/relscan.h"
 #include "hnsw.h"
 #include "pgstat.h"
@@ -26,6 +28,9 @@ GetScanItems(IndexScanDesc scan, Datum q)
 	/* Get m and entry point */
 	HnswGetMetaPageInfo(index, &m, &entryPoint);
 
+	so->q = q;
+	so->m = m;
+
 	if (entryPoint == NULL)
 		return NIL;
 
@@ -33,11 +38,44 @@ GetScanItems(IndexScanDesc scan, Datum q)
 
 	for (int lc = entryPoint->level; lc >= 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
+		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL, NULL, NULL, true, NULL);
 		ep = w;
 	}
 
-	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
+	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, &so->v, hnsw_streaming ? &so->discarded : NULL, true, &so->tuples);
+}
+
+/*
+ * Resume scan at ground level with discarded candidates
+ */
+static List *
+ResumeScanItems(IndexScanDesc scan)
+{
+	HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
+	Relation	index = scan->indexRelation;
+	FmgrInfo   *procinfo = so->procinfo;
+	Oid			collation = so->collation;
+	List	   *ep = NIL;
+	char	   *base = NULL;
+	int			batch_size = hnsw_ef_search;
+
+	if (pairingheap_is_empty(so->discarded))
+		return NIL;
+
+	/* Get next batch of candidates */
+	for (int i = 0; i < batch_size; i++)
+	{
+		HnswSearchCandidate *hc;
+
+		if (pairingheap_is_empty(so->discarded))
+			break;
+
+		hc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded));
+
+		ep = lappend(ep, hc);
+	}
+
+	return HnswSearchLayer(base, so->q, ep, batch_size, 0, index, procinfo, collation, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
 }
 
 /*
@@ -81,6 +119,8 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
 	so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
 	so->typeInfo = HnswGetTypeInfo(index);
 	so->first = true;
+	so->v.tids = NULL;
+	so->discarded = NULL;
 	so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
 									   "Hnsw scan temporary context",
 									   ALLOCSET_DEFAULT_SIZES);
@@ -103,7 +143,15 @@ hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int no
 {
 	HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
 
+	if (so->v.tids != NULL)
+		tidhash_reset(so->v.tids);
+
+	if (so->discarded != NULL)
+		pairingheap_reset(so->discarded);
+
 	so->first = true;
+	so->tuples = 0;
+	so->previousDistance = -INFINITY;
 	MemoryContextReset(so->tmpCtx);
 
 	if (keys && scan->numberOfKeys > 0)
@@ -153,34 +201,109 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 		 */
 		LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
 
-		so->w = GetScanItems(scan, value);
+		HnswBench("scan iteration", so->w = GetScanItems(scan, value));
 
 		/* Release shared lock */
 		UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
 
 		so->first = false;
 
-#if defined(HNSW_MEMORY) && PG_VERSION_NUM >= 130000
-		elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(so->tmpCtx, false) / (1024 * 1024));
+#if defined(HNSW_MEMORY)
+		elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
 #endif
 	}
 
-	while (list_length(so->w) > 0)
+	for (;;)
 	{
 		char	   *base = NULL;
-		HnswCandidate *hc = llast(so->w);
-		HnswElement element = HnswPtrAccess(base, hc->element);
+		HnswSearchCandidate *hc;
+		HnswElement element;
 		ItemPointer heaptid;
 
+		if (list_length(so->w) == 0)
+		{
+			if (!hnsw_streaming)
+				break;
+
+			/* Empty index */
+			if (so->discarded == NULL)
+				break;
+
+			/* Reached max number of additional tuples */
+			if (hnsw_ef_stream != -1 && so->tuples >= hnsw_ef_search + hnsw_ef_stream)
+			{
+				if (pairingheap_is_empty(so->discarded))
+					break;
+
+				/* Return remaining tuples */
+				so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
+			}
+			/* Prevent scans from consuming too much memory */
+			else if (MemoryContextMemAllocated(so->tmpCtx, false) > (Size) work_mem * 1024L)
+			{
+				if (pairingheap_is_empty(so->discarded))
+				{
+					ereport(NOTICE,
+							(errmsg("hnsw index scan exceeded work_mem after " INT64_FORMAT " tuples", so->tuples),
+							 errhint("Increase work_mem to scan more tuples.")));
+
+					break;
+				}
+
+				/* Return remaining tuples */
+				so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
+			}
+			else
+			{
+				/*
+				 * Locking ensures when neighbors are read, the elements they
+				 * reference will not be deleted (and replaced) during the
+				 * iteration.
+				 *
+				 * Elements loaded into memory on previous iterations may have
+				 * been deleted (and replaced), so when reading neighbors, the
+				 * element version must be checked.
+				 */
+				LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
+
+				HnswBench("scan iteration", so->w = ResumeScanItems(scan));
+
+				UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
+
+#if defined(HNSW_MEMORY)
+				elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
+#endif
+			}
+
+			if (list_length(so->w) == 0)
+				break;
+		}
+
+		hc = llast(so->w);
+		element = HnswPtrAccess(base, hc->element);
+
 		/* Move to next element if no valid heap TIDs */
 		if (element->heaptidsLength == 0)
 		{
 			so->w = list_delete_last(so->w);
+
+			/* Mark memory as free for next iteration */
+			if (hnsw_streaming)
+			{
+				pfree(element);
+				pfree(hc);
+			}
+
 			continue;
 		}
 
 		heaptid = &element->heaptids[--element->heaptidsLength];
 
+		if (hc->distance < so->previousDistance)
+			continue;
+
+		so->previousDistance = hc->distance;
+
 		MemoryContextSwitchTo(oldCtx);
 
 		scan->xs_heaptid = *heaptid;

src/hnswutils.c

@@ -5,6 +5,7 @@
 #include "access/generic_xlog.h"
 #include "catalog/pg_type.h"
 #include "catalog/pg_type_d.h"
+#include "common/hashfn.h"
 #include "fmgr.h"
 #include "hnsw.h"
 #include "lib/pairingheap.h"
@@ -14,12 +15,6 @@
 #include "utils/memdebug.h"
 #include "utils/rel.h"
 
-#if PG_VERSION_NUM >= 130000
-#include "common/hashfn.h"
-#else
-#include "utils/hashutils.h"
-#endif
-
 #if PG_VERSION_NUM < 170000
 static inline uint64
 murmurhash64(uint64 data)
@@ -107,10 +102,9 @@ hash_offset(Size offset)
 
 typedef union
 {
-	pointerhash_hash *pointers;
-	offsethash_hash *offsets;
-	tidhash_hash *tids;
-}			visited_hash;
+	HnswElement element;
+	ItemPointerData indextid;
+}			HnswUnvisited;
 
 /*
  * Get the max number of connections in an upper layer for each element in the index
@@ -252,6 +246,8 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
 
 	element->level = level;
 	element->deleted = 0;
+	/* Start at one to make it easier to find issues */
+	element->version = 1;
 
 	HnswInitNeighbors(base, element, m, allocator);
 
@@ -300,6 +296,9 @@ HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint)
 	page = BufferGetPage(buf);
 	metap = HnswPageGetMeta(page);
 
+	if (unlikely(metap->magicNumber != HNSW_MAGIC_NUMBER))
+		elog(ERROR, "hnsw index is not valid");
+
 	if (m != NULL)
 		*m = metap->m;
 
@@ -401,6 +400,7 @@ HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element)
 	etup->type = HNSW_ELEMENT_TUPLE_TYPE;
 	etup->level = element->level;
 	etup->deleted = 0;
+	etup->version = element->version;
 	for (int i = 0; i < HNSW_HEAPTIDS; i++)
 	{
 		if (i < element->heaptidsLength)
@@ -443,6 +443,7 @@ HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m)
 	}
 
 	ntup->count = idx;
+	ntup->version = e->version;
 }
 
 /*
@@ -516,6 +517,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
 {
 	element->level = etup->level;
 	element->deleted = etup->deleted;
+	element->version = etup->version;
 	element->neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
 	element->neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
 	element->heaptidsLength = 0;
@@ -544,19 +546,19 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
 /*
  * Load an element and optionally get its distance from q
  */
-void
-HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance)
+static void
+HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance, HnswElement * element)
 {
 	Buffer		buf;
 	Page		page;
 	HnswElementTuple etup;
 
 	/* Read vector */
-	buf = ReadBuffer(index, element->blkno);
+	buf = ReadBuffer(index, blkno);
 	LockBuffer(buf, BUFFER_LOCK_SHARE);
 	page = BufferGetPage(buf);
 
-	etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, element->offno));
+	etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
 
 	Assert(HnswIsElementTuple(etup));
 
@@ -571,19 +573,32 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
 
 	/* Load element */
 	if (distance == NULL || maxDistance == NULL || *distance < *maxDistance)
-		HnswLoadElementFromTuple(element, etup, true, loadVec);
+	{
+		if (*element == NULL)
+			*element = HnswInitElementFromBlock(blkno, offno);
+
+		HnswLoadElementFromTuple(*element, etup, true, loadVec);
+	}
 
 	UnlockReleaseBuffer(buf);
 }
 
 /*
- * Get the distance for a candidate
+ * Load an element and optionally get its distance from q
+ */
+void
+HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance)
+{
+	HnswLoadElementImpl(element->blkno, element->offno, distance, q, index, procinfo, collation, loadVec, maxDistance, &element);
+}
+
+/*
+ * Get the distance for an element
  */
 static float
-GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation)
+GetElementDistance(char *base, HnswElement element, Datum q, FmgrInfo *procinfo, Oid collation)
 {
-	HnswElement hce = HnswPtrAccess(base, hc->element);
-	Datum		value = HnswGetValue(base, hce);
+	Datum		value = HnswGetValue(base, element);
 
 	return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, value));
 }
@@ -591,14 +606,14 @@ GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, FmgrInfo *procinfo
 /*
  * Create a candidate for the entry point
  */
-HnswCandidate *
+HnswSearchCandidate *
 HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
 {
-	HnswCandidate *hc = palloc(sizeof(HnswCandidate));
+	HnswSearchCandidate *hc = palloc(sizeof(HnswSearchCandidate));
 
 	HnswPtrStore(base, hc->element, entryPoint);
 	if (index == NULL)
-		hc->distance = GetCandidateDistance(base, hc, q, procinfo, collation);
+		hc->distance = GetElementDistance(base, entryPoint, q, procinfo, collation);
 	else
 		HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec, NULL);
 	return hc;
@@ -610,10 +625,25 @@ HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index,
 static int
 CompareNearestCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
 {
-	if (((const HnswPairingHeapNode *) a)->inner->distance < ((const HnswPairingHeapNode *) b)->inner->distance)
+	if (HnswGetSearchCandidateConst(c_node, a)->distance < HnswGetSearchCandidateConst(c_node, b)->distance)
 		return 1;
 
-	if (((const HnswPairingHeapNode *) a)->inner->distance > ((const HnswPairingHeapNode *) b)->inner->distance)
+	if (HnswGetSearchCandidateConst(c_node, a)->distance > HnswGetSearchCandidateConst(c_node, b)->distance)
+		return -1;
+
+	return 0;
+}
+
+/*
+ * Compare discarded candidate distances
+ */
+static int
+CompareNearestDiscardedCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
+{
+	if (HnswGetSearchCandidateConst(w_node, a)->distance < HnswGetSearchCandidateConst(w_node, b)->distance)
+		return 1;
+
+	if (HnswGetSearchCandidateConst(w_node, a)->distance > HnswGetSearchCandidateConst(w_node, b)->distance)
 		return -1;
 
 	return 0;
@@ -625,27 +655,15 @@ CompareNearestCandidates(const pairingheap_node *a, const pairingheap_node *b, v
 static int
 CompareFurthestCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
 {
-	if (((const HnswPairingHeapNode *) a)->inner->distance < ((const HnswPairingHeapNode *) b)->inner->distance)
+	if (HnswGetSearchCandidateConst(w_node, a)->distance < HnswGetSearchCandidateConst(w_node, b)->distance)
 		return -1;
 
-	if (((const HnswPairingHeapNode *) a)->inner->distance > ((const HnswPairingHeapNode *) b)->inner->distance)
+	if (HnswGetSearchCandidateConst(w_node, a)->distance > HnswGetSearchCandidateConst(w_node, b)->distance)
 		return 1;
 
 	return 0;
 }
 
-/*
- * Create a pairing heap node for a candidate
- */
-static HnswPairingHeapNode *
-CreatePairingHeapNode(HnswCandidate * c)
-{
-	HnswPairingHeapNode *node = palloc(sizeof(HnswPairingHeapNode));
-
-	node->inner = c;
-	return node;
-}
-
 /*
  * Init visited
  */
@@ -664,11 +682,11 @@ InitVisited(char *base, visited_hash * v, Relation index, int ef, int m)
  * Add to visited
  */
 static inline void
-AddToVisited(char *base, visited_hash * v, HnswCandidate * hc, Relation index, bool *found)
+AddToVisited(char *base, visited_hash * v, HnswElementPtr elementPtr, Relation index, bool *found)
 {
 	if (index != NULL)
 	{
-		HnswElement element = HnswPtrAccess(base, hc->element);
+		HnswElement element = HnswPtrAccess(base, elementPtr);
 		ItemPointerData indextid;
 
 		ItemPointerSet(&indextid, element->blkno, element->offno);
@@ -676,23 +694,15 @@ AddToVisited(char *base, visited_hash * v, HnswCandidate * hc, Relation index, b
 	}
 	else if (base != NULL)
 	{
-#if PG_VERSION_NUM >= 130000
-		HnswElement element = HnswPtrAccess(base, hc->element);
+		HnswElement element = HnswPtrAccess(base, elementPtr);
 
-		offsethash_insert_hash(v->offsets, HnswPtrOffset(hc->element), element->hash, found);
-#else
-		offsethash_insert(v->offsets, HnswPtrOffset(hc->element), found);
-#endif
+		offsethash_insert_hash(v->offsets, HnswPtrOffset(elementPtr), element->hash, found);
 	}
 	else
 	{
-#if PG_VERSION_NUM >= 130000
-		HnswElement element = HnswPtrAccess(base, hc->element);
+		HnswElement element = HnswPtrAccess(base, elementPtr);
 
-		pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), element->hash, found);
-#else
-		pointerhash_insert(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), found);
-#endif
+		pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(elementPtr), element->hash, found);
 	}
 }
 
@@ -700,69 +710,168 @@ AddToVisited(char *base, visited_hash * v, HnswCandidate * hc, Relation index, b
  * Count element towards ef
  */
 static inline bool
-CountElement(char *base, HnswElement skipElement, HnswCandidate * hc)
+CountElement(HnswElement skipElement, HnswElement e)
 {
-	HnswElement e;
-
 	if (skipElement == NULL)
 		return true;
 
 	/* Ensure does not access heaptidsLength during in-memory build */
 	pg_memory_barrier();
 
-	e = HnswPtrAccess(base, hc->element);
+	/* Keep scan-build happy on Mac x86-64 */
+	Assert(e);
+
 	return e->heaptidsLength != 0;
 }
 
+/*
+ * Load unvisited neighbors from memory
+ */
+static void
+HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize)
+{
+	/* Get the neighborhood at layer lc */
+	HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc);
+
+	/* Copy neighborhood to local memory */
+	LWLockAcquire(&element->lock, LW_SHARED);
+	memcpy(localNeighborhood, neighborhood, neighborhoodSize);
+	LWLockRelease(&element->lock);
+
+	*unvisitedLength = 0;
+
+	for (int i = 0; i < localNeighborhood->length; i++)
+	{
+		HnswCandidate *hc = &localNeighborhood->items[i];
+		bool		found;
+
+		AddToVisited(base, v, hc->element, NULL, &found);
+
+		if (!found)
+			unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
+	}
+}
+
+/*
+ * Load unvisited neighbors from disk
+ */
+static void
+HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, Relation index, int m, int lm, int lc)
+{
+	Buffer		buf;
+	Page		page;
+	HnswNeighborTuple ntup;
+	int			start;
+	ItemPointerData indextids[HNSW_MAX_M * 2];
+
+	*unvisitedLength = 0;
+
+	buf = ReadBuffer(index, element->neighborPage);
+	LockBuffer(buf, BUFFER_LOCK_SHARE);
+	page = BufferGetPage(buf);
+
+	ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
+
+	/*
+	 * Ensure the neighbor tuple has not been deleted or replaced between
+	 * index scan iterations
+	 */
+	if (ntup->version != element->version || ntup->count != (element->level + 2) * m)
+	{
+		UnlockReleaseBuffer(buf);
+		return;
+	}
+
+	/* Copy to minimize lock time */
+	start = (element->level - lc) * m;
+	memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
+
+	UnlockReleaseBuffer(buf);
+
+	for (int i = 0; i < lm; i++)
+	{
+		ItemPointer indextid = &indextids[i];
+		bool		found;
+
+		if (!ItemPointerIsValid(indextid))
+			break;
+
+		tidhash_insert(v->tids, *indextid, &found);
+
+		if (!found)
+			unvisited[(*unvisitedLength)++].indextid = *indextid;
+	}
+}
+
 /*
  * Algorithm 2 from paper
  */
 List *
-HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
+HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples)
 {
 	List	   *w = NIL;
 	pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
 	pairingheap *W = pairingheap_allocate(CompareFurthestCandidates, NULL);
 	int			wlen = 0;
-	visited_hash v;
+	visited_hash vh;
 	ListCell   *lc2;
-	HnswNeighborArray *neighborhoodData = NULL;
-	Size		neighborhoodSize;
+	HnswNeighborArray *localNeighborhood = NULL;
+	Size		neighborhoodSize = 0;
+	int			lm = HnswGetLayerM(m, lc);
+	HnswUnvisited *unvisited = palloc(lm * sizeof(HnswUnvisited));
+	int			unvisitedLength;
 
-	InitVisited(base, &v, index, ef, m);
+	if (v == NULL)
+	{
+		v = &vh;
+		initVisited = true;
+	}
+
+	if (initVisited)
+	{
+		InitVisited(base, v, index, ef, m);
+
+		if (discarded != NULL)
+			*discarded = pairingheap_allocate(CompareNearestDiscardedCandidates, NULL);
+	}
 
 	/* Create local memory for neighborhood if needed */
 	if (index == NULL)
 	{
-		neighborhoodSize = HNSW_NEIGHBOR_ARRAY_SIZE(HnswGetLayerM(m, lc));
-		neighborhoodData = palloc(neighborhoodSize);
+		neighborhoodSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm);
+		localNeighborhood = palloc(neighborhoodSize);
 	}
 
 	/* Add entry points to v, C, and W */
 	foreach(lc2, ep)
 	{
-		HnswCandidate *hc = (HnswCandidate *) lfirst(lc2);
+		HnswSearchCandidate *hc = (HnswSearchCandidate *) lfirst(lc2);
 		bool		found;
 
-		AddToVisited(base, &v, hc, index, &found);
+		if (initVisited)
+		{
+			AddToVisited(base, v, hc->element, index, &found);
 
-		pairingheap_add(C, &(CreatePairingHeapNode(hc)->ph_node));
-		pairingheap_add(W, &(CreatePairingHeapNode(hc)->ph_node));
+			if (tuples != NULL)
+				(*tuples)++;
+		}
+
+		pairingheap_add(C, &hc->c_node);
+		pairingheap_add(W, &hc->w_node);
 
 		/*
 		 * Do not count elements being deleted towards ef when vacuuming. It
 		 * would be ideal to do this for inserts as well, but this could
 		 * affect insert performance.
 		 */
-		if (CountElement(base, skipElement, hc))
+		if (CountElement(skipElement, HnswPtrAccess(base, hc->element)))
 			wlen++;
 	}
 
 	while (!pairingheap_is_empty(C))
 	{
-		HnswNeighborArray *neighborhood;
-		HnswCandidate *c = ((HnswPairingHeapNode *) pairingheap_remove_first(C))->inner;
-		HnswCandidate *f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
+		HnswSearchCandidate *c = HnswGetSearchCandidate(c_node, pairingheap_remove_first(C));
+		HnswSearchCandidate *f = HnswGetSearchCandidate(w_node, pairingheap_first(W));
 		HnswElement cElement;
 
 		if (c->distance > f->distance)
@@ -770,71 +879,80 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 
 		cElement = HnswPtrAccess(base, c->element);
 
-		if (HnswPtrIsNull(base, cElement->neighbors))
-			HnswLoadNeighbors(cElement, index, m);
-
-		/* Get the neighborhood at layer lc */
-		neighborhood = HnswGetNeighbors(base, cElement, lc);
-
-		/* Copy neighborhood to local memory if needed */
 		if (index == NULL)
+			HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize);
+		else
+			HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
+
+		if (tuples != NULL)
+			(*tuples) += unvisitedLength;
+
+		for (int i = 0; i < unvisitedLength; i++)
 		{
-			LWLockAcquire(&cElement->lock, LW_SHARED);
-			memcpy(neighborhoodData, neighborhood, neighborhoodSize);
-			LWLockRelease(&cElement->lock);
-			neighborhood = neighborhoodData;
-		}
+			HnswElement eElement;
+			HnswSearchCandidate *e;
+			float		eDistance;
+			bool		alwaysAdd = wlen < ef;
 
-		for (int i = 0; i < neighborhood->length; i++)
-		{
-			HnswCandidate *e = &neighborhood->items[i];
-			bool		visited;
+			f = HnswGetSearchCandidate(w_node, pairingheap_first(W));
 
-			AddToVisited(base, &v, e, index, &visited);
-
-			if (!visited)
+			if (index == NULL)
 			{
-				float		eDistance;
-				HnswElement eElement = HnswPtrAccess(base, e->element);
+				eElement = unvisited[i].element;
+				eDistance = GetElementDistance(base, eElement, q, procinfo, collation);
+			}
+			else
+			{
+				ItemPointer indextid = &unvisited[i].indextid;
+				BlockNumber blkno = ItemPointerGetBlockNumber(indextid);
+				OffsetNumber offno = ItemPointerGetOffsetNumber(indextid);
 
-				f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
+				/* Avoid any allocations if not adding */
+				eElement = NULL;
+				HnswLoadElementImpl(blkno, offno, &eDistance, &q, index, procinfo, collation, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
+			}
 
-				if (index == NULL)
-					eDistance = GetCandidateDistance(base, e, q, procinfo, collation);
-				else
-					HnswLoadElement(eElement, &eDistance, &q, index, procinfo, collation, inserting, wlen >= ef ? &f->distance : NULL);
-
-				if (eDistance < f->distance || wlen < ef)
+			if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
+			{
+				if (discarded != NULL)
 				{
-					HnswCandidate *ec;
+					/* Create a new candidate */
+					e = palloc(sizeof(HnswSearchCandidate));
+					HnswPtrStore(base, e->element, eElement);
+					e->distance = eDistance;
+					pairingheap_add(*discarded, &e->w_node);
+				}
 
-					Assert(!eElement->deleted);
+				continue;
+			}
 
-					/* Make robust to issues */
-					if (eElement->level < lc)
-						continue;
+			/* Make robust to issues */
+			if (eElement->level < lc)
+				continue;
 
-					/* Copy e */
-					ec = palloc(sizeof(HnswCandidate));
-					HnswPtrStore(base, ec->element, eElement);
-					ec->distance = eDistance;
+			/* Create a new candidate */
+			e = palloc(sizeof(HnswSearchCandidate));
+			HnswPtrStore(base, e->element, eElement);
+			e->distance = eDistance;
+			pairingheap_add(C, &e->c_node);
+			pairingheap_add(W, &e->w_node);
 
-					pairingheap_add(C, &(CreatePairingHeapNode(ec)->ph_node));
-					pairingheap_add(W, &(CreatePairingHeapNode(ec)->ph_node));
+			/*
+			 * Do not count elements being deleted towards ef when vacuuming.
+			 * It would be ideal to do this for inserts as well, but this
+			 * could affect insert performance.
+			 */
+			if (CountElement(skipElement, eElement))
+			{
+				wlen++;
 
-					/*
-					 * Do not count elements being deleted towards ef when
-					 * vacuuming. It would be ideal to do this for inserts as
-					 * well, but this could affect insert performance.
-					 */
-					if (CountElement(base, skipElement, e))
-					{
-						wlen++;
+				/* No need to decrement wlen */
+				if (wlen > ef)
+				{
+					HnswSearchCandidate *d = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
 
-						/* No need to decrement wlen */
-						if (wlen > ef)
-							pairingheap_remove_first(W);
-					}
+					if (discarded != NULL)
+						pairingheap_add(*discarded, &d->w_node);
 				}
 			}
 		}
@@ -843,7 +961,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 	/* Add each element of W to w */
 	while (!pairingheap_is_empty(W))
 	{
-		HnswCandidate *hc = ((HnswPairingHeapNode *) pairingheap_remove_first(W))->inner;
+		HnswSearchCandidate *hc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
 
 		w = lappend(w, hc);
 	}
@@ -855,17 +973,10 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
  * Compare candidate distances with pointer tie-breaker
  */
 static int
-#if PG_VERSION_NUM >= 130000
 CompareCandidateDistances(const ListCell *a, const ListCell *b)
 {
 	HnswCandidate *hca = lfirst(a);
 	HnswCandidate *hcb = lfirst(b);
-#else
-CompareCandidateDistances(const void *a, const void *b)
-{
-	HnswCandidate *hca = lfirst(*(ListCell **) a);
-	HnswCandidate *hcb = lfirst(*(ListCell **) b);
-#endif
 
 	if (hca->distance < hcb->distance)
 		return 1;
@@ -886,17 +997,10 @@ CompareCandidateDistances(const void *a, const void *b)
  * Compare candidate distances with offset tie-breaker
  */
 static int
-#if PG_VERSION_NUM >= 130000
 CompareCandidateDistancesOffset(const ListCell *a, const ListCell *b)
 {
 	HnswCandidate *hca = lfirst(a);
 	HnswCandidate *hcb = lfirst(b);
-#else
-CompareCandidateDistancesOffset(const void *a, const void *b)
-{
-	HnswCandidate *hca = lfirst(*(ListCell **) a);
-	HnswCandidate *hcb = lfirst(*(ListCell **) b);
-#endif
 
 	if (hca->distance < hcb->distance)
 		return 1;
@@ -1106,7 +1210,7 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
 				if (HnswPtrIsNull(base, hc3Element->value))
 					HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true, NULL);
 				else
-					hc3->distance = GetCandidateDistance(base, hc3, q, procinfo, collation);
+					hc3->distance = GetElementDistance(base, hc3Element, q, procinfo, collation);
 
 				/* Prune element if being deleted */
 				if (hc3Element->heaptidsLength == 0)
@@ -1178,7 +1282,6 @@ RemoveElements(char *base, List *w, HnswElement skipElement)
 	return w2;
 }
 
-#if PG_VERSION_NUM >= 130000
 /*
  * Precompute hash
  */
@@ -1194,7 +1297,6 @@ PrecomputeHash(char *base, HnswElement element)
 	else
 		element->hash = hash_offset(HnswPtrOffset(ptr));
 }
-#endif
 
 /*
  * Algorithm 1 from paper
@@ -1209,11 +1311,9 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 	Datum		q = HnswGetValue(base, element);
 	HnswElement skipElement = existing ? element : NULL;
 
-#if PG_VERSION_NUM >= 130000
 	/* Precompute hash */
 	if (index == NULL)
 		PrecomputeHash(base, element);
-#endif
 
 	/* No neighbors if no entry point */
 	if (entryPoint == NULL)
@@ -1226,7 +1326,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 	/* 1st phase: greedy search to insert level */
 	for (int lc = entryLevel; lc >= level + 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
+		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement, NULL, NULL, true, NULL);
 		ep = w;
 	}
 
@@ -1242,16 +1342,27 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 	{
 		int			lm = HnswGetLayerM(m, lc);
 		List	   *neighbors;
-		List	   *lw;
+		List	   *lw = NIL;
+		ListCell   *lc2;
 
-		w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
+		w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement, NULL, NULL, true, NULL);
+
+		/* Convert search candidates to candidates */
+		foreach(lc2, w)
+		{
+			HnswSearchCandidate *sc = lfirst(lc2);
+			HnswCandidate *hc = palloc(sizeof(HnswCandidate));
+
+			hc->element = sc->element;
+			hc->distance = sc->distance;
+
+			lw = lappend(lw, hc);
+		}
 
 		/* Elements being deleted or skipped can help with search */
 		/* but should be removed before selecting neighbors */
 		if (index != NULL)
-			lw = RemoveElements(base, w, skipElement);
-		else
-			lw = w;
+			lw = RemoveElements(base, lw, skipElement);
 
 		/*
 		 * Candidates are sorted, but not deterministically. Could set
@@ -1276,7 +1387,9 @@ SparsevecCheckValue(Pointer v)
 	SparseVector *vec = (SparseVector *) v;
 
 	if (vec->nnz > HNSW_MAX_NNZ)
-		elog(ERROR, "sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ)));
 }
 
 /*
@@ -1301,7 +1414,7 @@ HnswGetTypeInfo(Relation index)
 		return (const HnswTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid));
 }
 
-PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_halfvec_support);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_halfvec_support);
 Datum
 hnsw_halfvec_support(PG_FUNCTION_ARGS)
 {
@@ -1314,7 +1427,7 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(&typeInfo);
 };
 
-PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_bit_support);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
 Datum
 hnsw_bit_support(PG_FUNCTION_ARGS)
 {
@@ -1327,7 +1440,7 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(&typeInfo);
 };
 
-PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
 Datum
 hnsw_sparsevec_support(PG_FUNCTION_ARGS)
 {

src/hnswvacuum.c

@@ -527,6 +527,14 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 			for (int i = 0; i < ntup->count; i++)
 				ItemPointerSetInvalid(&ntup->indextids[i]);
 
+			/* Increment version */
+			/* This is used to avoid incorrect reads for iterative scans */
+			/* Reserve some bits for future use */
+			etup->version++;
+			if (etup->version > 15)
+				etup->version = 1;
+			ntup->version = etup->version;
+
 			/*
 			 * We modified the tuples in place, no need to call
 			 * PageIndexTupleOverwrite

src/ivfbuild.c

@@ -26,12 +26,6 @@
 #include "pgstat.h"
 #endif
 
-#if PG_VERSION_NUM >= 130000
-#define CALLBACK_ITEM_POINTER ItemPointer tid
-#else
-#define CALLBACK_ITEM_POINTER HeapTuple hup
-#endif
-
 #if PG_VERSION_NUM >= 140000
 #include "utils/backend_status.h"
 #include "utils/wait_event.h"
@@ -96,7 +90,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
  * Callback for sampling
  */
 static void
-SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
+SampleCallback(Relation index, ItemPointer tid, Datum *values,
 			   bool *isnull, bool tupleIsAlive, void *state)
 {
 	IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
@@ -207,16 +201,12 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
  * Callback for table_index_build_scan
  */
 static void
-BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
+BuildCallback(Relation index, ItemPointer tid, Datum *values,
 			  bool *isnull, bool tupleIsAlive, void *state)
 {
 	IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
 	MemoryContext oldCtx;
 
-#if PG_VERSION_NUM < 130000
-	ItemPointer tid = &hup->t_self;
-#endif
-
 	/* Skip nulls */
 	if (isnull[0])
 		return;
@@ -335,14 +325,20 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 
 	/* Disallow varbit since require fixed dimensions */
 	if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
-		elog(ERROR, "type not supported for ivfflat index");
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("type not supported for ivfflat index")));
 
 	/* Require column to have dimensions to be indexed */
 	if (buildstate->dimensions < 0)
-		elog(ERROR, "column does not have dimensions");
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("column does not have dimensions")));
 
 	if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
-		elog(ERROR, "column cannot have more than %d dimensions for ivfflat index", buildstate->typeInfo->maxDimensions);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("column cannot have more than %d dimensions for ivfflat index", buildstate->typeInfo->maxDimensions)));
 
 	buildstate->reltuples = 0;
 	buildstate->indtuples = 0;
@@ -355,7 +351,9 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 
 	/* Require more than one dimension for spherical k-means */
 	if (buildstate->kmeansnormprocinfo != NULL && buildstate->dimensions == 1)
-		elog(ERROR, "dimensions must be greater than one for this opclass");
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("dimensions must be greater than one for this opclass")));
 
 	/* Create tuple description for sorting */
 	buildstate->tupdesc = CreateTemplateTupleDesc(3);
@@ -562,6 +560,20 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
 }
 #endif
 
+/*
+ * Initialize build sort state
+ */
+static Tuplesortstate *
+InitBuildSortState(TupleDesc tupdesc, int memory, SortCoordinate coordinate)
+{
+	AttrNumber	attNums[] = {1};
+	Oid			sortOperators[] = {Int4LessOperator};
+	Oid			sortCollations[] = {InvalidOid};
+	bool		nullsFirstFlags[] = {false};
+
+	return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, memory, coordinate, false);
+}
+
 /*
  * Within leader, wait for end of heap scan
  */
@@ -609,12 +621,6 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
 	double		reltuples;
 	IndexInfo  *indexInfo;
 
-	/* Sort options, which must match AssignTuples */
-	AttrNumber	attNums[] = {1};
-	Oid			sortOperators[] = {Int4LessOperator};
-	Oid			sortCollations[] = {InvalidOid};
-	bool		nullsFirstFlags[] = {false};
-
 	/* Initialize local tuplesort coordination state */
 	coordinate = palloc0(sizeof(SortCoordinateData));
 	coordinate->isWorker = true;
@@ -627,7 +633,7 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
 	InitBuildState(&buildstate, ivfspool->heap, ivfspool->index, indexInfo);
 	memcpy(buildstate.centers->items, ivfcenters, buildstate.centers->itemsize * buildstate.centers->maxlen);
 	buildstate.centers->length = buildstate.centers->maxlen;
-	ivfspool->sortstate = tuplesort_begin_heap(buildstate.tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, sortmem, coordinate, false);
+	ivfspool->sortstate = InitBuildSortState(buildstate.tupdesc, sortmem, coordinate);
 	buildstate.sortstate = ivfspool->sortstate;
 	scan = table_beginscan_parallel(ivfspool->heap,
 									ParallelTableScanFromIvfflatShared(ivfshared));
@@ -924,12 +930,6 @@ AssignTuples(IvfflatBuildState * buildstate)
 	int			parallel_workers = 0;
 	SortCoordinate coordinate = NULL;
 
-	/* Sort options, which must match IvfflatParallelScanAndSort */
-	AttrNumber	attNums[] = {1};
-	Oid			sortOperators[] = {Int4LessOperator};
-	Oid			sortCollations[] = {InvalidOid};
-	bool		nullsFirstFlags[] = {false};
-
 	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN);
 
 	/* Calculate parallel workers */
@@ -950,7 +950,7 @@ AssignTuples(IvfflatBuildState * buildstate)
 	}
 
 	/* Begin serial/leader tuplesort */
-	buildstate->sortstate = tuplesort_begin_heap(buildstate->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, maintenance_work_mem, coordinate, false);
+	buildstate->sortstate = InitBuildSortState(buildstate->tupdesc, maintenance_work_mem, coordinate);
 
 	/* Add tuples to sort */
 	if (buildstate->heap != NULL)
@@ -1006,6 +1006,10 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
 	CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
 	CreateEntryPages(buildstate, forkNum);
 
+	/* Write WAL for initialization fork since GenericXLog functions do not */
+	if (forkNum == INIT_FORKNUM)
+		log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
+
 	FreeBuildState(buildstate);
 }
 

src/ivfflat.c

@@ -7,6 +7,7 @@
 #include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "ivfflat.h"
+#include "utils/float.h"
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
 #include "utils/spccache.h"
@@ -26,11 +27,7 @@ IvfflatInit(void)
 {
 	ivfflat_relopt_kind = add_reloption_kind();
 	add_int_reloption(ivfflat_relopt_kind, "lists", "Number of inverted lists",
-					  IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS
-#if PG_VERSION_NUM >= 130000
-					  ,AccessExclusiveLock
-#endif
-		);
+					  IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, AccessExclusiveLock);
 
 	DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
 							"Valid range is 1..lists.", &ivfflat_probes,
@@ -78,8 +75,8 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	/* Never use index without order */
 	if (path->indexorderbys == NULL)
 	{
-		*indexStartupCost = DBL_MAX;
-		*indexTotalCost = DBL_MAX;
+		*indexStartupCost = get_float8_infinity();
+		*indexTotalCost = get_float8_infinity();
 		*indexSelectivity = 0;
 		*indexCorrelation = 0;
 		*indexPages = 0;
@@ -123,13 +120,6 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 		costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
 	}
 
-	/*
-	 * If the list selectivity is lower than what is returned from the generic
-	 * cost estimator, use that.
-	 */
-	if (ratio < costs.indexSelectivity)
-		costs.indexSelectivity = ratio;
-
 	/* Use total cost since most work happens before first tuple is returned */
 	*indexStartupCost = costs.indexTotalCost;
 	*indexTotalCost = costs.indexTotalCost;
@@ -148,23 +138,10 @@ ivfflatoptions(Datum reloptions, bool validate)
 		{"lists", RELOPT_TYPE_INT, offsetof(IvfflatOptions, lists)},
 	};
 
-#if PG_VERSION_NUM >= 130000
 	return (bytea *) build_reloptions(reloptions, validate,
 									  ivfflat_relopt_kind,
 									  sizeof(IvfflatOptions),
 									  tab, lengthof(tab));
-#else
-	relopt_value *options;
-	int			numoptions;
-	IvfflatOptions *rdopts;
-
-	options = parseRelOptions(reloptions, validate, ivfflat_relopt_kind, &numoptions);
-	rdopts = allocateReloptStruct(sizeof(IvfflatOptions), options, numoptions);
-	fillRelOptions((void *) rdopts, sizeof(IvfflatOptions), options, numoptions,
-				   validate, tab, lengthof(tab));
-
-	return (bytea *) rdopts;
-#endif
 }
 
 /*
@@ -181,7 +158,7 @@ ivfflatvalidate(Oid opclassoid)
  *
  * See https://www.postgresql.org/docs/current/index-api.html
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
 Datum
 ivfflathandler(PG_FUNCTION_ARGS)
 {
@@ -189,9 +166,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
 
 	amroutine->amstrategies = 0;
 	amroutine->amsupport = 5;
-#if PG_VERSION_NUM >= 130000
 	amroutine->amoptsprocnum = 0;
-#endif
 	amroutine->amcanorder = false;
 	amroutine->amcanorderbyop = true;
 	amroutine->amcanbackward = false;	/* can change direction mid-scan */
@@ -204,17 +179,24 @@ ivfflathandler(PG_FUNCTION_ARGS)
 	amroutine->amclusterable = false;
 	amroutine->ampredlocks = false;
 	amroutine->amcanparallel = false;
-	amroutine->amcaninclude = false;
-#if PG_VERSION_NUM >= 130000
-	amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
-	amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
+#if PG_VERSION_NUM >= 170000
+	amroutine->amcanbuildparallel = true;
 #endif
+	amroutine->amcaninclude = false;
+	amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
+#if PG_VERSION_NUM >= 160000
+	amroutine->amsummarizing = false;
+#endif
+	amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
 	amroutine->amkeytype = InvalidOid;
 
 	/* Interface functions */
 	amroutine->ambuild = ivfflatbuild;
 	amroutine->ambuildempty = ivfflatbuildempty;
 	amroutine->aminsert = ivfflatinsert;
+#if PG_VERSION_NUM >= 170000
+	amroutine->aminsertcleanup = NULL;
+#endif
 	amroutine->ambulkdelete = ivfflatbulkdelete;
 	amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
 	amroutine->amcanreturn = NULL;	/* tuple not included in heapsort */

src/ivfflat.h

@@ -253,8 +253,9 @@ typedef struct IvfflatScanOpaqueData
 	/* Sorting */
 	Tuplesortstate *sortstate;
 	TupleDesc	tupdesc;
-	TupleTableSlot *slot;
-	bool		isnull;
+	TupleTableSlot *vslot;
+	TupleTableSlot *mslot;
+	BufferAccessStrategy bas;
 
 	/* Support functions */
 	FmgrInfo   *procinfo;

src/ivfinsert.c

@@ -94,6 +94,9 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
 		value = IvfflatNormValue(typeInfo, collation, value);
 	}
 
+	/* Ensure index is valid */
+	IvfflatGetMetaPageInfo(index, NULL, NULL);
+
 	/* Find the insert page - sets the page and list info */
 	FindInsertPage(index, values, &insertPage, &listInfo);
 	Assert(BlockNumberIsValid(insertPage));

src/ivfkmeans.c

@@ -151,12 +151,8 @@ RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeI
 static void
 ShowMemoryUsage(MemoryContext context, Size estimatedSize)
 {
-#if PG_VERSION_NUM >= 130000
 	elog(INFO, "total memory: %zu MB",
 		 MemoryContextMemAllocated(context, true) / (1024 * 1024));
-#else
-	MemoryContextStats(context);
-#endif
 	elog(INFO, "estimated memory: %zu MB", estimatedSize / (1024 * 1024));
 }
 #endif
@@ -327,7 +323,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
 	newCenters->length = numCenters;
 
 #ifdef IVFFLAT_MEMORY
-	ShowMemoryUsage(MemoryContextGetParent(CurrentMemoryContext));
+	ShowMemoryUsage(MemoryContextGetParent(CurrentMemoryContext), totalSize);
 #endif
 
 	/* Pick initial centers */

src/ivfscan.c

@@ -11,16 +11,23 @@
 #include "pgstat.h"
 #include "storage/bufmgr.h"
 
+#ifdef IVFFLAT_MEMORY
+#include "utils/memutils.h"
+#endif
+
+#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
+#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
+
 /*
  * Compare list distances
  */
 static int
 CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg)
 {
-	if (((const IvfflatScanList *) a)->distance > ((const IvfflatScanList *) b)->distance)
+	if (GetScanListConst(a)->distance > GetScanListConst(b)->distance)
 		return 1;
 
-	if (((const IvfflatScanList *) a)->distance < ((const IvfflatScanList *) b)->distance)
+	if (GetScanListConst(a)->distance < GetScanListConst(b)->distance)
 		return -1;
 
 	return 0;
@@ -72,14 +79,14 @@ GetScanLists(IndexScanDesc scan, Datum value)
 
 				/* Calculate max distance */
 				if (listCount == so->probes)
-					maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
+					maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
 			}
 			else if (distance < maxDistance)
 			{
 				IvfflatScanList *scanlist;
 
 				/* Remove */
-				scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
+				scanlist = GetScanList(pairingheap_remove_first(so->listQueue));
 
 				/* Reuse */
 				scanlist->startPage = list->startPage;
@@ -87,7 +94,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
 				pairingheap_add(so->listQueue, &scanlist->ph_node);
 
 				/* Update max distance */
-				maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
+				maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
 			}
 		}
 
@@ -106,19 +113,12 @@ GetScanItems(IndexScanDesc scan, Datum value)
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
 	TupleDesc	tupdesc = RelationGetDescr(scan->indexRelation);
 	double		tuples = 0;
-	TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
-
-	/*
-	 * Reuse same set of shared buffers for scan
-	 *
-	 * See postgres/src/backend/storage/buffer/README for description
-	 */
-	BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
+	TupleTableSlot *slot = so->vslot;
 
 	/* Search closest probes lists */
 	while (!pairingheap_is_empty(so->listQueue))
 	{
-		BlockNumber searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
+		BlockNumber searchPage = GetScanList(pairingheap_remove_first(so->listQueue))->startPage;
 
 		/* Search all entry pages for list */
 		while (BlockNumberIsValid(searchPage))
@@ -127,7 +127,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
 			Page		page;
 			OffsetNumber maxoffno;
 
-			buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, bas);
+			buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, so->bas);
 			LockBuffer(buf, BUFFER_LOCK_SHARE);
 			page = BufferGetPage(buf);
 			maxoffno = PageGetMaxOffsetNumber(page);
@@ -166,8 +166,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
 		}
 	}
 
-	FreeAccessStrategy(bas);
-
 	if (tuples < 100)
 		ereport(DEBUG1,
 				(errmsg("index scan found few tuples"),
@@ -217,6 +215,20 @@ GetScanValue(IndexScanDesc scan)
 	return value;
 }
 
+/*
+ * Initialize scan sort state
+ */
+static Tuplesortstate *
+InitScanSortState(TupleDesc tupdesc)
+{
+	AttrNumber	attNums[] = {1};
+	Oid			sortOperators[] = {Float8LessOperator};
+	Oid			sortCollations[] = {InvalidOid};
+	bool		nullsFirstFlags[] = {false};
+
+	return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
+}
+
 /*
  * Prepare for an index scan
  */
@@ -227,10 +239,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	IvfflatScanOpaque so;
 	int			lists;
 	int			dimensions;
-	AttrNumber	attNums[] = {1};
-	Oid			sortOperators[] = {Float8LessOperator};
-	Oid			sortCollations[] = {InvalidOid};
-	bool		nullsFirstFlags[] = {false};
 	int			probes = ivfflat_probes;
 
 	scan = RelationGetIndexScan(index, nkeys, norderbys);
@@ -258,9 +266,18 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
 
 	/* Prep sort */
-	so->sortstate = tuplesort_begin_heap(so->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
+	so->sortstate = InitScanSortState(so->tupdesc);
 
-	so->slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
+	/* Need separate slots for puttuple and gettuple */
+	so->vslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
+	so->mslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
+
+	/*
+	 * Reuse same set of shared buffers for scan
+	 *
+	 * See postgres/src/backend/storage/buffer/README for description
+	 */
+	so->bas = GetAccessStrategy(BAS_BULKREAD);
 
 	so->listQueue = pairingheap_allocate(CompareLists, scan);
 
@@ -277,10 +294,8 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
 {
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
 
-#if PG_VERSION_NUM >= 130000
 	if (!so->first)
 		tuplesort_reset(so->sortstate);
-#endif
 
 	so->first = true;
 	pairingheap_reset(so->listQueue);
@@ -327,14 +342,19 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
 		IvfflatBench("GetScanItems", GetScanItems(scan, value));
 		so->first = false;
 
+#if defined(IVFFLAT_MEMORY)
+		elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
+#endif
+
 		/* Clean up if we allocated a new value */
 		if (value != scan->orderByData->sk_argument)
 			pfree(DatumGetPointer(value));
 	}
 
-	if (tuplesort_gettupleslot(so->sortstate, true, false, so->slot, NULL))
+	if (tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
 	{
-		ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
+		bool		isnull;
+		ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
 
 		scan->xs_heaptid = *heaptid;
 		scan->xs_recheck = false;
@@ -355,6 +375,10 @@ ivfflatendscan(IndexScanDesc scan)
 
 	pairingheap_free(so->listQueue);
 	tuplesort_end(so->sortstate);
+	FreeAccessStrategy(so->bas);
+	FreeTupleDesc(so->tupdesc);
+
+	/* TODO Free vslot and mslot without freeing TupleDesc */
 
 	pfree(so);
 	scan->opaque = NULL;

src/ivfutils.c

@@ -170,7 +170,11 @@ IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions)
 	page = BufferGetPage(buf);
 	metap = IvfflatPageGetMeta(page);
 
-	*lists = metap->lists;
+	if (unlikely(metap->magicNumber != IVFFLAT_MAGIC_NUMBER))
+		elog(ERROR, "ivfflat index is not valid");
+
+	if (lists != NULL)
+		*lists = metap->lists;
 
 	if (dimensions != NULL)
 		*dimensions = metap->dimensions;
@@ -338,7 +342,7 @@ IvfflatGetTypeInfo(Relation index)
 		return (const IvfflatTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid));
 }
 
-PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflat_halfvec_support);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_halfvec_support);
 Datum
 ivfflat_halfvec_support(PG_FUNCTION_ARGS)
 {
@@ -353,7 +357,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(&typeInfo);
 };
 
-PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflat_bit_support);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
 Datum
 ivfflat_bit_support(PG_FUNCTION_ARGS)
 {

src/sparsevec.c

@@ -3,6 +3,7 @@
 #include <limits.h>
 #include <math.h>
 
+#include "catalog/pg_type.h"
 #include "common/string.h"
 #include "fmgr.h"
 #include "halfutils.h"
@@ -11,6 +12,7 @@
 #include "sparsevec.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
+#include "utils/lsyscache.h"
 #include "vector.h"
 
 #if PG_VERSION_NUM >= 120000
@@ -188,7 +190,7 @@ CompareIndices(const void *a, const void *b)
 /*
  * Convert textual representation to internal representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_in);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_in);
 Datum
 sparsevec_in(PG_FUNCTION_ARGS)
 {
@@ -409,7 +411,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
 /*
  * Convert internal representation to textual representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_out);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_out);
 Datum
 sparsevec_out(PG_FUNCTION_ARGS)
 {
@@ -462,7 +464,7 @@ sparsevec_out(PG_FUNCTION_ARGS)
 /*
  * Convert type modifier
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
 Datum
 sparsevec_typmod_in(PG_FUNCTION_ARGS)
 {
@@ -493,7 +495,7 @@ sparsevec_typmod_in(PG_FUNCTION_ARGS)
 /*
  * Convert external binary representation to internal representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_recv);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_recv);
 Datum
 sparsevec_recv(PG_FUNCTION_ARGS)
 {
@@ -545,7 +547,7 @@ sparsevec_recv(PG_FUNCTION_ARGS)
 /*
  * Convert internal representation to the external binary representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_send);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_send);
 Datum
 sparsevec_send(PG_FUNCTION_ARGS)
 {
@@ -572,7 +574,7 @@ sparsevec_send(PG_FUNCTION_ARGS)
  * Convert sparse vector to sparse vector
  * This is needed to check the type modifier
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec);
 Datum
 sparsevec(PG_FUNCTION_ARGS)
 {
@@ -587,7 +589,7 @@ sparsevec(PG_FUNCTION_ARGS)
 /*
  * Convert dense vector to sparse vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_sparsevec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_sparsevec);
 Datum
 vector_to_sparsevec(PG_FUNCTION_ARGS)
 {
@@ -630,7 +632,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
 /*
  * Convert half vector to sparse vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_sparsevec);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_sparsevec);
 Datum
 halfvec_to_sparsevec(PG_FUNCTION_ARGS)
 {
@@ -670,6 +672,137 @@ halfvec_to_sparsevec(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(result);
 }
 
+/*
+ * Convert array to sparse vector
+ */
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_sparsevec);
+Datum
+array_to_sparsevec(PG_FUNCTION_ARGS)
+{
+	ArrayType  *array = PG_GETARG_ARRAYTYPE_P(0);
+	int32		typmod = PG_GETARG_INT32(1);
+	SparseVector *result;
+	int16		typlen;
+	bool		typbyval;
+	char		typalign;
+	Datum	   *elemsp;
+	int			nelemsp;
+	int			nnz = 0;
+	float	   *values;
+	int			j = 0;
+
+	if (ARR_NDIM(array) > 1)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_EXCEPTION),
+				 errmsg("array must be 1-D")));
+
+	if (ARR_HASNULL(array) && array_contains_nulls(array))
+		ereport(ERROR,
+				(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
+				 errmsg("array must not contain nulls")));
+
+	get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
+	deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, NULL, &nelemsp);
+
+	CheckDim(nelemsp);
+	CheckExpectedDim(typmod, nelemsp);
+
+#ifdef _MSC_VER
+/* /fp:fast may not propagate +/-Infinity or NaN */
+#define IS_NOT_ZERO(v) (isnan((float) (v)) || isinf((float) (v)) || ((float) (v)) != 0)
+#else
+#define IS_NOT_ZERO(v) (((float) (v)) != 0)
+#endif
+
+	if (ARR_ELEMTYPE(array) == INT4OID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			nnz += IS_NOT_ZERO(DatumGetInt32(elemsp[i]));
+	}
+	else if (ARR_ELEMTYPE(array) == FLOAT8OID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			nnz += IS_NOT_ZERO(DatumGetFloat8(elemsp[i]));
+	}
+	else if (ARR_ELEMTYPE(array) == FLOAT4OID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			nnz += IS_NOT_ZERO(DatumGetFloat4(elemsp[i]));
+	}
+	else if (ARR_ELEMTYPE(array) == NUMERICOID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			nnz += IS_NOT_ZERO(DirectFunctionCall1(numeric_float4, elemsp[i]));
+	}
+	else
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_EXCEPTION),
+				 errmsg("unsupported array type")));
+	}
+
+	result = InitSparseVector(nelemsp, nnz);
+	values = SPARSEVEC_VALUES(result);
+
+#define PROCESS_ARRAY_ELEM(elem) \
+	do { \
+		float v = (float) (elem); \
+		if (IS_NOT_ZERO(v)) { \
+			/* Safety check */ \
+			if (j >= result->nnz) \
+				elog(ERROR, "safety check failed"); \
+			result->indices[j] = i; \
+			values[j] = v; \
+			j++; \
+		} \
+	} while (0)
+
+	if (ARR_ELEMTYPE(array) == INT4OID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			PROCESS_ARRAY_ELEM(DatumGetInt32(elemsp[i]));
+	}
+	else if (ARR_ELEMTYPE(array) == FLOAT8OID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			PROCESS_ARRAY_ELEM(DatumGetFloat8(elemsp[i]));
+	}
+	else if (ARR_ELEMTYPE(array) == FLOAT4OID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			PROCESS_ARRAY_ELEM(DatumGetFloat4(elemsp[i]));
+	}
+	else if (ARR_ELEMTYPE(array) == NUMERICOID)
+	{
+		for (int i = 0; i < nelemsp; i++)
+			PROCESS_ARRAY_ELEM(DatumGetFloat4(DirectFunctionCall1(numeric_float4, elemsp[i])));
+	}
+	else
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_EXCEPTION),
+				 errmsg("unsupported array type")));
+	}
+
+#undef PROCESS_ARRAY_ELEM
+#undef IS_NOT_ZERO
+
+	/*
+	 * Free allocation from deconstruct_array. Do not free individual elements
+	 * when pass-by-reference since they point to original array.
+	 */
+	pfree(elemsp);
+
+	if (j != result->nnz)
+		elog(ERROR, "correctness check failed");
+
+	/* Check elements */
+	for (int i = 0; i < result->nnz; i++)
+		CheckElement(values[i]);
+
+	PG_RETURN_POINTER(result);
+}
+
 /*
  * Get the L2 squared distance between sparse vectors
  */
@@ -721,7 +854,7 @@ SparsevecL2SquaredDistance(SparseVector * a, SparseVector * b)
 /*
  * Get the L2 distance between sparse vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_distance);
 Datum
 sparsevec_l2_distance(PG_FUNCTION_ARGS)
 {
@@ -737,7 +870,7 @@ sparsevec_l2_distance(PG_FUNCTION_ARGS)
  * Get the L2 squared distance between sparse vectors
  * This saves a sqrt calculation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance);
 Datum
 sparsevec_l2_squared_distance(PG_FUNCTION_ARGS)
 {
@@ -788,7 +921,7 @@ SparsevecInnerProduct(SparseVector * a, SparseVector * b)
 /*
  * Get the inner product of two sparse vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_inner_product);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_inner_product);
 Datum
 sparsevec_inner_product(PG_FUNCTION_ARGS)
 {
@@ -803,7 +936,7 @@ sparsevec_inner_product(PG_FUNCTION_ARGS)
 /*
  * Get the negative inner product of two sparse vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product);
 Datum
 sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
 {
@@ -818,7 +951,7 @@ sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
 /*
  * Get the cosine distance between two sparse vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cosine_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_cosine_distance);
 Datum
 sparsevec_cosine_distance(PG_FUNCTION_ARGS)
 {
@@ -863,7 +996,7 @@ sparsevec_cosine_distance(PG_FUNCTION_ARGS)
 /*
  * Get the L1 distance between two sparse vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l1_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l1_distance);
 Datum
 sparsevec_l1_distance(PG_FUNCTION_ARGS)
 {
@@ -912,7 +1045,7 @@ sparsevec_l1_distance(PG_FUNCTION_ARGS)
 /*
  * Get the L2 norm of a sparse vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_norm);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_norm);
 Datum
 sparsevec_l2_norm(PG_FUNCTION_ARGS)
 {
@@ -930,7 +1063,7 @@ sparsevec_l2_norm(PG_FUNCTION_ARGS)
 /*
  * Normalize a sparse vector with the L2 norm
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_normalize);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_normalize);
 Datum
 sparsevec_l2_normalize(PG_FUNCTION_ARGS)
 {
@@ -1040,7 +1173,7 @@ sparsevec_cmp_internal(SparseVector * a, SparseVector * b)
 /*
  * Less than
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_lt);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_lt);
 Datum
 sparsevec_lt(PG_FUNCTION_ARGS)
 {
@@ -1053,7 +1186,7 @@ sparsevec_lt(PG_FUNCTION_ARGS)
 /*
  * Less than or equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_le);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_le);
 Datum
 sparsevec_le(PG_FUNCTION_ARGS)
 {
@@ -1066,7 +1199,7 @@ sparsevec_le(PG_FUNCTION_ARGS)
 /*
  * Equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_eq);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_eq);
 Datum
 sparsevec_eq(PG_FUNCTION_ARGS)
 {
@@ -1079,7 +1212,7 @@ sparsevec_eq(PG_FUNCTION_ARGS)
 /*
  * Not equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_ne);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_ne);
 Datum
 sparsevec_ne(PG_FUNCTION_ARGS)
 {
@@ -1092,7 +1225,7 @@ sparsevec_ne(PG_FUNCTION_ARGS)
 /*
  * Greater than or equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_ge);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_ge);
 Datum
 sparsevec_ge(PG_FUNCTION_ARGS)
 {
@@ -1105,7 +1238,7 @@ sparsevec_ge(PG_FUNCTION_ARGS)
 /*
  * Greater than
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_gt);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_gt);
 Datum
 sparsevec_gt(PG_FUNCTION_ARGS)
 {
@@ -1118,7 +1251,7 @@ sparsevec_gt(PG_FUNCTION_ARGS)
 /*
  * Compare sparse vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cmp);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_cmp);
 Datum
 sparsevec_cmp(PG_FUNCTION_ARGS)
 {

src/vector.c

@@ -26,11 +26,6 @@
 #include "varatt.h"
 #endif
 
-#if PG_VERSION_NUM < 130000
-#define TYPALIGN_DOUBLE 'd'
-#define TYPALIGN_INT 'i'
-#endif
-
 #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
 #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
 
@@ -160,28 +155,10 @@ CheckStateArray(ArrayType *statearray, const char *caller)
 	return (float8 *) ARR_DATA_PTR(statearray);
 }
 
-#if PG_VERSION_NUM < 120003
-static pg_noinline void
-float_overflow_error(void)
-{
-	ereport(ERROR,
-			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
-			 errmsg("value out of range: overflow")));
-}
-
-static pg_noinline void
-float_underflow_error(void)
-{
-	ereport(ERROR,
-			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
-			 errmsg("value out of range: underflow")));
-}
-#endif
-
 /*
  * Convert textual representation to internal representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_in);
 Datum
 vector_in(PG_FUNCTION_ARGS)
 {
@@ -294,7 +271,7 @@ vector_in(PG_FUNCTION_ARGS)
 /*
  * Convert internal representation to textual representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_out);
 Datum
 vector_out(PG_FUNCTION_ARGS)
 {
@@ -348,7 +325,7 @@ PrintVector(char *msg, Vector * vector)
 /*
  * Convert type modifier
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_typmod_in);
 Datum
 vector_typmod_in(PG_FUNCTION_ARGS)
 {
@@ -379,7 +356,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
 /*
  * Convert external binary representation to internal representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_recv);
 Datum
 vector_recv(PG_FUNCTION_ARGS)
 {
@@ -413,7 +390,7 @@ vector_recv(PG_FUNCTION_ARGS)
 /*
  * Convert internal representation to the external binary representation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_send);
 Datum
 vector_send(PG_FUNCTION_ARGS)
 {
@@ -433,7 +410,7 @@ vector_send(PG_FUNCTION_ARGS)
  * Convert vector to vector
  * This is needed to check the type modifier
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector);
 Datum
 vector(PG_FUNCTION_ARGS)
 {
@@ -448,7 +425,7 @@ vector(PG_FUNCTION_ARGS)
 /*
  * Convert array to vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_vector);
 Datum
 array_to_vector(PG_FUNCTION_ARGS)
 {
@@ -522,7 +499,7 @@ array_to_vector(PG_FUNCTION_ARGS)
 /*
  * Convert vector to float4[]
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_float4);
 Datum
 vector_to_float4(PG_FUNCTION_ARGS)
 {
@@ -546,7 +523,7 @@ vector_to_float4(PG_FUNCTION_ARGS)
 /*
  * Convert half vector to vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_vector);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_vector);
 Datum
 halfvec_to_vector(PG_FUNCTION_ARGS)
 {
@@ -584,7 +561,7 @@ VectorL2SquaredDistance(int dim, float *ax, float *bx)
 /*
  * Get the L2 distance between vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_distance);
 Datum
 l2_distance(PG_FUNCTION_ARGS)
 {
@@ -600,7 +577,7 @@ l2_distance(PG_FUNCTION_ARGS)
  * Get the L2 squared distance between vectors
  * This saves a sqrt calculation
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
 Datum
 vector_l2_squared_distance(PG_FUNCTION_ARGS)
 {
@@ -627,7 +604,7 @@ VectorInnerProduct(int dim, float *ax, float *bx)
 /*
  * Get the inner product of two vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(inner_product);
 Datum
 inner_product(PG_FUNCTION_ARGS)
 {
@@ -642,7 +619,7 @@ inner_product(PG_FUNCTION_ARGS)
 /*
  * Get the negative inner product of two vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_negative_inner_product);
 Datum
 vector_negative_inner_product(PG_FUNCTION_ARGS)
 {
@@ -676,7 +653,7 @@ VectorCosineSimilarity(int dim, float *ax, float *bx)
 /*
  * Get the cosine distance between two vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(cosine_distance);
 Datum
 cosine_distance(PG_FUNCTION_ARGS)
 {
@@ -708,7 +685,7 @@ cosine_distance(PG_FUNCTION_ARGS)
  * Currently uses angular distance since needs to satisfy triangle inequality
  * Assumes inputs are unit vectors (skips norm)
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_spherical_distance);
 Datum
 vector_spherical_distance(PG_FUNCTION_ARGS)
 {
@@ -745,7 +722,7 @@ VectorL1Distance(int dim, float *ax, float *bx)
 /*
  * Get the L1 distance between two vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(l1_distance);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l1_distance);
 Datum
 l1_distance(PG_FUNCTION_ARGS)
 {
@@ -760,7 +737,7 @@ l1_distance(PG_FUNCTION_ARGS)
 /*
  * Get the dimensions of a vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_dims);
 Datum
 vector_dims(PG_FUNCTION_ARGS)
 {
@@ -772,7 +749,7 @@ vector_dims(PG_FUNCTION_ARGS)
 /*
  * Get the L2 norm of a vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_norm);
 Datum
 vector_norm(PG_FUNCTION_ARGS)
 {
@@ -790,7 +767,7 @@ vector_norm(PG_FUNCTION_ARGS)
 /*
  * Normalize a vector with the L2 norm
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_normalize);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_normalize);
 Datum
 l2_normalize(PG_FUNCTION_ARGS)
 {
@@ -829,7 +806,7 @@ l2_normalize(PG_FUNCTION_ARGS)
 /*
  * Add vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_add);
 Datum
 vector_add(PG_FUNCTION_ARGS)
 {
@@ -862,7 +839,7 @@ vector_add(PG_FUNCTION_ARGS)
 /*
  * Subtract vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_sub);
 Datum
 vector_sub(PG_FUNCTION_ARGS)
 {
@@ -895,7 +872,7 @@ vector_sub(PG_FUNCTION_ARGS)
 /*
  * Multiply vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_mul);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_mul);
 Datum
 vector_mul(PG_FUNCTION_ARGS)
 {
@@ -931,7 +908,7 @@ vector_mul(PG_FUNCTION_ARGS)
 /*
  * Concatenate vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_concat);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_concat);
 Datum
 vector_concat(PG_FUNCTION_ARGS)
 {
@@ -955,7 +932,7 @@ vector_concat(PG_FUNCTION_ARGS)
 /*
  * Quantize a vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(binary_quantize);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(binary_quantize);
 Datum
 binary_quantize(PG_FUNCTION_ARGS)
 {
@@ -973,7 +950,7 @@ binary_quantize(PG_FUNCTION_ARGS)
 /*
  * Get a subvector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(subvector);
 Datum
 subvector(PG_FUNCTION_ARGS)
 {
@@ -1047,7 +1024,7 @@ vector_cmp_internal(Vector * a, Vector * b)
 /*
  * Less than
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_lt);
 Datum
 vector_lt(PG_FUNCTION_ARGS)
 {
@@ -1060,7 +1037,7 @@ vector_lt(PG_FUNCTION_ARGS)
 /*
  * Less than or equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_le);
 Datum
 vector_le(PG_FUNCTION_ARGS)
 {
@@ -1073,7 +1050,7 @@ vector_le(PG_FUNCTION_ARGS)
 /*
  * Equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_eq);
 Datum
 vector_eq(PG_FUNCTION_ARGS)
 {
@@ -1086,7 +1063,7 @@ vector_eq(PG_FUNCTION_ARGS)
 /*
  * Not equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ne);
 Datum
 vector_ne(PG_FUNCTION_ARGS)
 {
@@ -1099,7 +1076,7 @@ vector_ne(PG_FUNCTION_ARGS)
 /*
  * Greater than or equal
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ge);
 Datum
 vector_ge(PG_FUNCTION_ARGS)
 {
@@ -1112,7 +1089,7 @@ vector_ge(PG_FUNCTION_ARGS)
 /*
  * Greater than
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_gt);
 Datum
 vector_gt(PG_FUNCTION_ARGS)
 {
@@ -1125,7 +1102,7 @@ vector_gt(PG_FUNCTION_ARGS)
 /*
  * Compare vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_cmp);
 Datum
 vector_cmp(PG_FUNCTION_ARGS)
 {
@@ -1138,7 +1115,7 @@ vector_cmp(PG_FUNCTION_ARGS)
 /*
  * Accumulate vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_accum);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_accum);
 Datum
 vector_accum(PG_FUNCTION_ARGS)
 {
@@ -1199,7 +1176,7 @@ vector_accum(PG_FUNCTION_ARGS)
 /*
  * Combine vectors or half vectors (also used for halfvec_combine)
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_combine);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_combine);
 Datum
 vector_combine(PG_FUNCTION_ARGS)
 {
@@ -1270,7 +1247,7 @@ vector_combine(PG_FUNCTION_ARGS)
 /*
  * Average vectors
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_avg);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_avg);
 Datum
 vector_avg(PG_FUNCTION_ARGS)
 {
@@ -1304,7 +1281,7 @@ vector_avg(PG_FUNCTION_ARGS)
 /*
  * Convert sparse vector to dense vector
  */
-PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_vector);
+FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_vector);
 Datum
 sparsevec_to_vector(PG_FUNCTION_ARGS)
 {

src/vector.h

@@ -20,4 +20,11 @@ Vector	   *InitVector(int dim);
 void		PrintVector(char *msg, Vector * vector);
 int			vector_cmp_internal(Vector * a, Vector * b);
 
+/* TODO Move to better place */
+#if PG_VERSION_NUM >= 160000
+#define FUNCTION_PREFIX
+#else
+#define FUNCTION_PREFIX PGDLLEXPORT
+#endif
+
 #endif

test/expected/cast.out

@@ -208,6 +208,62 @@ SELECT '{1:1e-8}/1'::sparsevec::halfvec;
  [0]
 (1 row)
 
+SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
+      array      
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
+      array      
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
+      array      
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
+      array      
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
+      array      
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
+    sparsevec    
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
+    sparsevec    
+-----------------
+ {1:1,3:2,5:3}/6
+(1 row)
+
+SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
+ERROR:  expected 5 dimensions, not 6
+SELECT '{NULL}'::real[]::sparsevec;
+ERROR:  array must not contain nulls
+SELECT '{NaN}'::real[]::sparsevec;
+ERROR:  NaN not allowed in sparsevec
+SELECT '{Infinity}'::real[]::sparsevec;
+ERROR:  infinite value not allowed in sparsevec
+SELECT '{-Infinity}'::real[]::sparsevec;
+ERROR:  infinite value not allowed in sparsevec
+SELECT '{}'::real[]::sparsevec;
+ERROR:  sparsevec must have at least 1 dimension
+SELECT '{{1}}'::real[]::sparsevec;
+ERROR:  array must be 1-D
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
 ERROR:  vector cannot have more than 16000 dimensions
 SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

test/perl/PostgreSQL/Test/Cluster.pm

@@ -0,0 +1,11 @@
+package PostgreSQL::Test::Cluster;
+
+use PostgresNode;
+
+sub new
+{
+	my ($class, $name) = @_;
+	return get_new_node($name);
+}
+
+1;

test/perl/PostgreSQL/Test/Utils.pm

@@ -0,0 +1,5 @@
+package PostgreSQL::Test::Utils;
+
+use TestLib;
+
+1;

test/perl/PostgresNode.pm

@@ -1,8 +0,0 @@
-use PostgreSQL::Test::Cluster;
-
-sub get_new_node
-{
-	return PostgreSQL::Test::Cluster->new(@_);
-}
-
-1;

test/perl/TestLib.pm

@@ -1,3 +0,0 @@
-use PostgreSQL::Test::Utils;
-
-1;

test/sql/cast.sql

@@ -58,6 +58,22 @@ SELECT '{}/16001'::sparsevec::halfvec;
 SELECT '{1:65520}/1'::sparsevec::halfvec;
 SELECT '{1:1e-8}/1'::sparsevec::halfvec;
 
+SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
+SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
+SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
+SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
+SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
+
+SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
+SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
+SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
+SELECT '{NULL}'::real[]::sparsevec;
+SELECT '{NaN}'::real[]::sparsevec;
+SELECT '{Infinity}'::real[]::sparsevec;
+SELECT '{-Infinity}'::real[]::sparsevec;
+SELECT '{}'::real[]::sparsevec;
+SELECT '{{1}}'::real[]::sparsevec;
+
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
 SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
 

test/t/001_ivfflat_wal.pl

@@ -2,9 +2,9 @@
 
 # Test generic xlog record work for ivfflat index replication.
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 32;
@@ -49,7 +49,7 @@ sub test_index_replay
 my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize primary node
-$node_primary = get_new_node('primary');
+$node_primary = PostgreSQL::Test::Cluster->new('primary');
 $node_primary->init(allows_streaming => 1);
 if ($dim > 32)
 {
@@ -67,7 +67,7 @@ my $backup_name = 'my_backup';
 $node_primary->backup($backup_name);
 
 # Create streaming replica linking to primary
-$node_replica = get_new_node('replica');
+$node_replica = PostgreSQL::Test::Cluster->new('replica');
 $node_replica->init_from_backup($node_primary, $backup_name, has_streaming => 1);
 $node_replica->start;
 

test/t/002_ivfflat_vacuum.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 3;
@@ -15,7 +15,7 @@ for (1 .. $dim)
 my $array_sql = join(", ", @r);
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/003_ivfflat_vector_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -49,7 +49,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/004_ivfflat_vector_insert_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -48,7 +48,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/005_ivfflat_query_recall.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/006_ivfflat_lists.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/007_ivfflat_inserts.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 768;
@@ -9,7 +9,7 @@ my $dim = 768;
 my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/008_ivfflat_centers.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/009_ivfflat_filtering.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 3;
@@ -11,7 +11,7 @@ my $limit = 20;
 my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 
@@ -94,8 +94,7 @@ like($explain, qr/Seq Scan/);
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query';
 ));
-# TODO Do not use index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Seq Scan/);
 
 # Test attribute index
 $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
@@ -110,7 +109,6 @@ $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING ivfflat (v v
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Use partial index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Index Scan using partial_idx/);
 
 done_testing();

test/t/010_hnsw_wal.pl

@@ -2,9 +2,9 @@
 
 # Test generic xlog record work for hnsw index replication.
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 32;
@@ -49,7 +49,7 @@ sub test_index_replay
 my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize primary node
-$node_primary = get_new_node('primary');
+$node_primary = PostgreSQL::Test::Cluster->new('primary');
 $node_primary->init(allows_streaming => 1);
 if ($dim > 32)
 {
@@ -67,7 +67,7 @@ my $backup_name = 'my_backup';
 $node_primary->backup($backup_name);
 
 # Create streaming replica linking to primary
-$node_replica = get_new_node('replica');
+$node_replica = PostgreSQL::Test::Cluster->new('replica');
 $node_replica->init_from_backup($node_primary, $backup_name, has_streaming => 1);
 $node_replica->start;
 

test/t/011_hnsw_vacuum.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 3;
@@ -15,7 +15,7 @@ for (1 .. $dim)
 my $array_sql = join(", ", @r);
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/012_hnsw_vector_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -47,7 +47,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/013_hnsw_vector_insert_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -47,7 +47,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/014_hnsw_vector_vacuum_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -48,7 +48,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/015_hnsw_vector_duplicates.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/016_hnsw_inserts.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Ensures elements and neighbors on both same and different pages
@@ -10,7 +10,7 @@ my $dim = 1900;
 my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/017_hnsw_filtering.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 3;
@@ -11,16 +11,20 @@ my $limit = 20;
 my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 
 # Create table and index
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
 $node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);");
+$node->safe_psql("postgres", "CREATE TABLE cat (i int4 PRIMARY KEY, t text, b boolean);");
 $node->safe_psql("postgres",
 	"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;"
 );
+$node->safe_psql("postgres",
+	"INSERT INTO cat SELECT i, 'cat ' || i, i % 5 = 0 FROM generate_series(1, $nc) i;"
+);
 $node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
 $node->safe_psql("postgres", "ANALYZE tst;");
 
@@ -96,13 +100,25 @@ $explain = $node->safe_psql("postgres", qq(
 ));
 like($explain, qr/Seq Scan/);
 
+# Test join
+$explain = $node->safe_psql("postgres", qq(
+	EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c ORDER BY v <-> '$query' LIMIT $limit;
+));
+like($explain, qr/Index Scan using idx/);
+
+# Test join with attribute filtering
+$explain = $node->safe_psql("postgres", qq(
+	EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c WHERE cat.b = 't' ORDER BY v <-> '$query' LIMIT $limit;
+));
+like($explain, qr/Index Scan using idx/);
+
 # Test attribute index
 $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Use attribute index
-like($explain, qr/Index Scan using idx/);
+# Use attribute index
+like($explain, qr/Bitmap Index Scan on attribute_idx/);
 
 # Test partial index
 $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);");

test/t/018_aggregates.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 
@@ -53,7 +53,7 @@ sub test_aggregate
 	else
 	{
 		# Does not raise overflow error in this instance due to loss of precision
-		is($res, "[24576,24576,49152]")
+		is($res, "[24576,24576,49152]");
 	}
 }
 

test/t/019_storage.pl

@@ -1,13 +1,13 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 1024;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/020_hnsw_bit_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -51,7 +51,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/021_hnsw_bit_insert_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -51,7 +51,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/022_hnsw_bit_vacuum_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -51,7 +51,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/023_hnsw_bit_duplicates.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/024_hnsw_halfvec_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -48,7 +48,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/025_hnsw_halfvec_insert_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -48,7 +48,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/026_hnsw_halfvec_vacuum_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -48,7 +48,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/027_hnsw_halfvec_duplicates.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/028_hnsw_sparsevec_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -47,7 +47,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/029_hnsw_sparsevec_insert_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -47,7 +47,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/030_hnsw_sparsevec_vacuum_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -48,7 +48,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/031_hnsw_sparsevec_duplicates.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/032_ivfflat_halfvec_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -51,7 +51,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/033_comparison.pl

@@ -1,14 +1,14 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
 my $array_sql = join(",", ('floor(random() * 2)::int - 1') x 3);
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/034_distance_functions.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -10,7 +10,7 @@ my $dim = 5;
 my $array_sql = join(",", ('floor(random() * 4)::int - 2') x $dim);
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/035_ivfflat_bit_build_recall.pl

@@ -1,7 +1,7 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $node;
@@ -51,7 +51,7 @@ sub test_recall
 }
 
 # Initialize node
-$node = get_new_node('node');
+$node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/036_ivfflat_bit_centers.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/037_inputs.pl

@@ -1,11 +1,11 @@
 use strict;
-use warnings;
-use PostgresNode;
-use TestLib;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
 use Test::More;
 
 # Initialize node
-my $node = get_new_node('node');
+my $node = PostgreSQL::Test::Cluster->new('node');
 $node->init;
 $node->start;
 

test/t/038_hnsw_sparsevec_vacuum_insert.pl

@@ -0,0 +1,42 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Initialize node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+# Create table and index
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v sparsevec(100000));");
+$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v sparsevec_l2_ops);");
+
+for (1 .. 3)
+{
+	for (1 .. 100)
+	{
+		my @elements;
+		my %indices;
+		for (1 .. int(rand() * 100))
+		{
+			my $index = int(rand() * (100000 - 1)) + 1;
+			if (!exists($indices{$index}))
+			{
+				my $value = rand();
+				push(@elements, "$index:$value");
+				$indices{$index} = 1;
+			}
+		}
+		my $embedding = "{" . join(",", @elements) . "}/100000";
+		$node->safe_psql("postgres", "INSERT INTO tst (v) VALUES ('$embedding');");
+	}
+
+	$node->safe_psql("postgres", "DELETE FROM tst WHERE i % 2 = 0;");
+	$node->safe_psql("postgres", "VACUUM tst;");
+	is(1, 1);
+}
+
+done_testing();

test/t/039_hnsw_cost.pl

@@ -0,0 +1,46 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my @dims = (384, 1536);
+my $limit = 10;
+
+# Initialize node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+
+for my $dim (@dims)
+{
+	my $array_sql = join(",", ('random()') x $dim);
+	my $n = $dim == 384 ? 2000 : 1000;
+
+	# Create table and index
+	$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
+	$node->safe_psql("postgres",
+		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, $n) i;"
+	);
+	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
+	$node->safe_psql("postgres", "ANALYZE tst;");
+
+	# Generate query
+	my @r = ();
+	for (1 .. $dim)
+	{
+		push(@r, rand());
+	}
+	my $query = "[" . join(",", @r) . "]";
+
+	my $explain = $node->safe_psql("postgres", qq(
+		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx/);
+
+	$node->safe_psql("postgres", "DROP TABLE tst;");
+}
+
+done_testing();

test/t/040_ivfflat_cost.pl

@@ -0,0 +1,45 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my @dims = (384, 1536);
+my $limit = 10;
+
+# Initialize node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+
+for my $dim (@dims)
+{
+	my $array_sql = join(",", ('random()') x $dim);
+
+	# Create table and index
+	$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
+	$node->safe_psql("postgres",
+		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 5000) i;"
+	);
+	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5);");
+	$node->safe_psql("postgres", "ANALYZE tst;");
+
+	# Generate query
+	my @r = ();
+	for (1 .. $dim)
+	{
+		push(@r, rand());
+	}
+	my $query = "[" . join(",", @r) . "]";
+
+	my $explain = $node->safe_psql("postgres", qq(
+		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx/);
+
+	$node->safe_psql("postgres", "DROP TABLE tst;");
+}
+
+done_testing();

test/t/041_hnsw_streaming.pl

@@ -0,0 +1,66 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $dim = 3;
+my $array_sql = join(",", ('random()') x $dim);
+
+# Initialize node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+# Create table
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+$node->safe_psql("postgres", "CREATE TABLE tst (i int4 PRIMARY KEY, v vector($dim));");
+$node->safe_psql("postgres",
+	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
+);
+$node->safe_psql("postgres", qq(
+	SET maintenance_work_mem = '128MB';
+	SET max_parallel_maintenance_workers = 2;
+	CREATE INDEX ON tst USING hnsw (v vector_l2_ops)
+));
+
+my $count = $node->safe_psql("postgres", qq(
+	SET enable_seqscan = off;
+	SET hnsw.streaming = on;
+	SET work_mem = '8MB';
+	SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
+));
+is($count, 10);
+
+foreach ((30000, 50000, 70000))
+{
+	my $ef_stream = $_;
+	my $expected = $ef_stream / 10000;
+	my $sum = 0;
+
+	for my $i (1 .. 20)
+	{
+		$count = $node->safe_psql("postgres", qq(
+			SET enable_seqscan = off;
+			SET hnsw.streaming = on;
+			SET hnsw.ef_stream = $ef_stream;
+			SET work_mem = '8MB';
+			SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst WHERE i = $i) LIMIT 11) t;
+		));
+		$sum += $count;
+	}
+
+	my $avg = $sum / 20;
+	cmp_ok($avg, '>', $expected - 2);
+	cmp_ok($avg, '<', $expected + 2);
+}
+
+my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
+	SET enable_seqscan = off;
+	SET hnsw.streaming = on;
+	SET work_mem = '2MB';
+	SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
+));
+like($stderr, qr/hnsw index scan exceeded work_mem after \d+ tuples/);
+
+done_testing();

test/t/042_hnsw_streaming_recall.pl

@@ -0,0 +1,131 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node;
+my @queries = ();
+my @expected;
+my $limit = 20;
+my $dim = 3;
+my $array_sql = join(",", ('random()') x $dim);
+my @cs = (100, 1000);
+
+sub test_recall
+{
+	my ($c, $ef_search, $min, $operator) = @_;
+	my $correct = 0;
+	my $total = 0;
+
+	my $explain = $node->safe_psql("postgres", qq(
+		SET enable_seqscan = off;
+		SET hnsw.ef_search = $ef_search;
+		SET hnsw.streaming = on;
+		EXPLAIN ANALYZE SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[0]' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx on tst/);
+
+	for my $i (0 .. $#queries)
+	{
+		my $actual = $node->safe_psql("postgres", qq(
+			SET enable_seqscan = off;
+			SET hnsw.ef_search = $ef_search;
+			SET hnsw.streaming = on;
+			SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[$i]' LIMIT $limit;
+		));
+		my @actual_ids = split("\n", $actual);
+
+		my @expected_ids = split("\n", $expected[$i]);
+		my %expected_set = map { $_ => 1 } @expected_ids;
+
+		foreach (@actual_ids)
+		{
+			if (exists($expected_set{$_}))
+			{
+				$correct++;
+			}
+		}
+
+		$total += $limit;
+	}
+
+	cmp_ok($correct / $total, ">=", $min, $operator);
+}
+
+# Initialize node
+$node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+# Create table
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
+$node->safe_psql("postgres",
+	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
+);
+
+# Generate queries
+for (1 .. 20)
+{
+	my @r = ();
+	for (1 .. $dim)
+	{
+		push(@r, rand());
+	}
+	push(@queries, "[" . join(",", @r) . "]");
+}
+
+# Check each index type
+my @operators = ("<->", "<=>");
+my @opclasses = ("vector_l2_ops", "vector_cosine_ops");
+
+for my $i (0 .. $#operators)
+{
+	my $operator = $operators[$i];
+	my $opclass = $opclasses[$i];
+
+	$node->safe_psql("postgres", qq(
+		SET maintenance_work_mem = '128MB';
+		CREATE INDEX idx ON tst USING hnsw (v $opclass);
+	));
+
+	foreach (@cs)
+	{
+		my $c = $_;
+
+		# Get exact results
+		@expected = ();
+		foreach (@queries)
+		{
+			my $res = $node->safe_psql("postgres", qq(
+				SET enable_indexscan = off;
+				WITH top AS (
+					SELECT v $operator '$_' AS distance FROM tst WHERE i % $c = 0 ORDER BY distance LIMIT $limit
+				)
+				SELECT i FROM tst WHERE (v $operator '$_') <= (SELECT MAX(distance) FROM top)
+			));
+			push(@expected, $res);
+		}
+
+		if ($c == 100)
+		{
+			test_recall($c, 40, 0.99, $operator);
+		}
+		else
+		{
+			if ($operator eq "<->")
+			{
+				test_recall($c, 40, 0.99, $operator);
+			}
+			else
+			{
+				test_recall($c, 40, 0.99, $operator);
+			}
+		}
+	}
+
+	$node->safe_psql("postgres", "DROP INDEX idx;");
+}
+
+done_testing();

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat and hnsw access methods'
-default_version = '0.7.1'
+default_version = '0.7.4'
 module_pathname = '$libdir/vector'
 relocatable = true