Added HnswGetElementTupleSize method

Previously, the cost estimation formula for a HNSW index scan utilized
a methodology that only factored in the entry level for an HNSW scan
and the "m" index parameter, which reflects the number of tuples (or
vectors) to scan at each step of a HNSW graph traversal. While this
would bias the PostgreSQL query planner to choose an HNSW index scan
over other available paths, this could lead to potential suboptimal
index selection, for example, choosing to use a HNSW index instead of
an available B-tree index that has better selectivity.

The number of tuples scanned during HNSW graph traversal is principally
influenced by these factors:

 * The number of tuples stored in the index
 * `m` - the number of tuples that are scanned in each step of the graph
   traversal
 * `hnsw.ef_search` - which influences the total number of steps it
   takes for the scan to converge on the approximated nearest neighbors

Through testing different source models for vectors, we also observed
that the correlation of vectors in mdoels would impact this convergence.
For this first iteration, we've opted to hardcode a constant scaling
factor and set it to `0.55`, though a future commit may turn this into
a configurable parameter.

The high-level formula for estimating the cost of a HNSW index scan is
as such:

```
(entryLevel * m) + (layer0TuplesMax * layer0Selectivity)
```

where

- `(entryLevel * m)` is the lower bound of tuples to scan, as it
accounts for the graph traversal to layer 0 (L0). (L1 and above has an ef=1)
- `layer0TuplesMax` is an estimate of the maximum number of tuples to
scan at L0. This accounts for tuples that may end up being discarded due
to them already being visited. Testing shows that the number of steps
until converge is similar to the value of `hnsw.ef_search`, thus we can
estimate tuples max at `hnsw.ef_search * m * 2`
- `layer0Selectivity` - estimates the percentage of tuples that will
actually be scanned during the index traversal, multipled by the scaling
factor

In addition to the `m` build parameter and `hsnw.ef_search`, costs
estimates can be influenced by standard PostgreSQL costing parameters,
though adjusting those (e.g. `random_page_cost`) should be done with
care.

Co-authored-by: @ankane

CHANGELOG.md

@@ -1,6 +1,9 @@
 ## 0.8.0 (unreleased)
 
+- Added support for inline filtering with HNSW
 - Added casts for arrays to `sparsevec`
+- Improved cost estimation
+- Improved performance of HNSW inserts and on-disk index builds
 - Reduced memory usage for HNSW index scans
 - Dropped support for Postgres 12
 

Dockerfile

@@ -1,4 +1,4 @@
-ARG PG_MAJOR=16
+ARG PG_MAJOR=17
 FROM postgres:$PG_MAJOR
 ARG PG_MAJOR
 

Makefile

@@ -4,8 +4,8 @@ EXTVERSION = 0.7.4
 MODULE_big = vector
 DATA = $(wildcard sql/*--*--*.sql)
 DATA_built = sql/$(EXTENSION)--$(EXTVERSION).sql
-OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/minivec.o src/ivfvacuum.o src/sparsevec.o src/vector.o
-HEADERS = src/halfvec.h src/minivec.h src/sparsevec.h src/vector.h
+OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
+HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
 
 TESTS = $(wildcard test/sql/*.sql)
 REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
@@ -66,7 +66,7 @@ dist:
 	git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master
 
 # for Docker
-PG_MAJOR ?= 16
+PG_MAJOR ?= 17
 
 .PHONY: docker
 

Makefile.win

@@ -2,10 +2,10 @@ EXTENSION = vector
 EXTVERSION = 0.7.4
 
 DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
-OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\minivec.obj src\sparsevec.obj src\vector.obj
-HEADERS = src\halfvec.h src\minivec.h src\sparsevec.h src\vector.h
+OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
+HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
 
-REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector minivec sparsevec vector_type
+REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type
 REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
 
 # For /arch flags

README.md

@@ -52,6 +52,8 @@ nmake /F Makefile.win
 nmake /F Makefile.win install
 ```
 
+Note: Postgres 17 is not supported yet due to an upstream issue
+
 See the [installation notes](#installation-notes---windows) if you run into issues
 
 You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -100,6 +102,8 @@ Or add a vector column to an existing table
 ALTER TABLE items ADD COLUMN embedding vector(3);
 ```
 
+Also supports [half-precision](#half-precision-vectors), [binary](#binary-vectors), and [sparse](#sparse-vectors) vectors
+
 Insert vectors
 
 ```sql
@@ -145,6 +149,8 @@ Supported distance functions are:
 - `<#>` - (negative) inner product
 - `<=>` - cosine distance
 - `<+>` - L1 distance (added in 0.7.0)
+- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
+- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
 
 Get the nearest neighbors to a row
 
@@ -433,6 +439,12 @@ Create an index on one [or more](https://www.postgresql.org/docs/current/indexes
 CREATE INDEX ON items (category_id);
 ```
 
+Or a composite HNSW index for approximate search (added in 0.8.0)
+
+```sql
+CREATE INDEX ON items USING hnsw (embedding vector_l2_ops, category_id);
+```
+
 Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
 
 ```sql
@@ -934,37 +946,6 @@ Function | Description | Added
 avg(halfvec) → halfvec | average | 0.7.0
 sum(halfvec) → halfvec | sum | 0.7.0
 
-### Minivec Type
-
-Each mini vector takes `dimensions + 8` bytes of storage. Each element is a E4M3 8-bit floating-point number, and all elements must be finite (no `NaN`). Mini vectors can have up to 16,000 dimensions.
-
-### Minivec Operators
-
-Operator | Description | Added
---- | --- | ---
-\+ | element-wise addition | 0.8.0
-\- | element-wise subtraction | 0.8.0
-\* | element-wise multiplication | 0.8.0
-\|\| | concatenate | 0.8.0
-<-> | Euclidean distance | 0.8.0
-<#> | negative inner product | 0.8.0
-<=> | cosine distance | 0.8.0
-<+> | taxicab distance | 0.8.0
-
-### Minivec Functions
-
-Function | Description | Added
---- | --- | ---
-binary_quantize(minivec) → bit | binary quantize | 0.8.0
-cosine_distance(minivec, minivec) → double precision | cosine distance | 0.8.0
-inner_product(minivec, minivec) → double precision | inner product | 0.8.0
-l1_distance(minivec, minivec) → double precision | taxicab distance | 0.8.0
-l2_distance(minivec, minivec) → double precision | Euclidean distance | 0.8.0
-l2_norm(minivec) → double precision | Euclidean norm | 0.8.0
-l2_normalize(minivec) → minivec | Normalize with Euclidean norm | 0.8.0
-subvector(minivec, integer, integer) → minivec | subvector | 0.8.0
-vector_dims(minivec) → integer | number of dimensions | 0.8.0
-
 ### Bit Type
 
 Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres docs](https://www.postgresql.org/docs/current/datatype-bit.html) for more info.
@@ -1014,7 +995,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
 If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
 
 ```sh
-export PG_CONFIG=/Library/PostgreSQL/16/bin/pg_config
+export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
 ```
 
 Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1025,11 +1006,11 @@ sudo --preserve-env=PG_CONFIG make install
 
 A few common paths on Mac are:
 
-- EDB installer - `/Library/PostgreSQL/16/bin/pg_config`
-- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@16/bin/pg_config`
-- Homebrew (x86-64) - `/usr/local/opt/postgresql@16/bin/pg_config`
+- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
+- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
+- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
 
-Note: Replace `16` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### Missing Header
 
@@ -1038,10 +1019,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
 For Ubuntu and Debian, use:
 
 ```sh
-sudo apt install postgresql-server-dev-16
+sudo apt install postgresql-server-dev-17
 ```
 
-Note: Replace `16` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### Missing SDK
 
@@ -1074,17 +1055,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
 Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
 
 ```sh
-docker pull pgvector/pgvector:pg16
+docker pull pgvector/pgvector:pg17
 ```
 
-This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
+This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
 
 You can also build the image manually:
 
 ```sh
 git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
-docker build --pull --build-arg PG_MAJOR=16 -t myuser/pgvector .
+docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
 ```
 
 ### Homebrew
@@ -1095,7 +1076,7 @@ With Homebrew Postgres, you can use:
 brew install pgvector
 ```
 
-Note: This only adds it to the `postgresql@14` formula
+Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
 
 ### PGXN
 
@@ -1110,22 +1091,22 @@ pgxn install vector
 Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
 
 ```sh
-sudo apt install postgresql-16-pgvector
+sudo apt install postgresql-17-pgvector
 ```
 
-Note: Replace `16` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### Yum
 
 RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
 
 ```sh
-sudo yum install pgvector_16
+sudo yum install pgvector_17
 # or
-sudo dnf install pgvector_16
+sudo dnf install pgvector_17
 ```
 
-Note: Replace `16` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### pkg
 
@@ -1214,6 +1195,7 @@ Thanks to:
 - [k-means++: The Advantage of Careful Seeding](https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf)
 - [Concept Decompositions for Large Sparse Text Data using Clustering](https://www.cs.utexas.edu/users/inderjit/public_papers/concept_mlj.pdf)
 - [Efficient and Robust Approximate Nearest Neighbor Search using Hierarchical Navigable Small World Graphs](https://arxiv.org/ftp/arxiv/papers/1603/1603.09320.pdf)
+- [HQANN: Efficient and Robust Similarity Search for Hybrid Queries with Structured and Unstructured Constraints](https://arxiv.org/pdf/2207.07940.pdf)
 
 ## History
 

sql/vector--0.7.4--0.8.0.sql

@@ -1,8 +1,6 @@
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "ALTER EXTENSION vector UPDATE TO '0.8.0'" to load this file. \quit
 
--- TODO minivec functions
-
 CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
 	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
 
@@ -26,3 +24,11 @@ CREATE CAST (double precision[] AS sparsevec)
 
 CREATE CAST (numeric[] AS sparsevec)
 	WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;
+
+CREATE FUNCTION hnsw_attribute_distance(integer, integer) RETURNS float8
+	AS 'MODULE_PATHNAME', 'hnsw_int4_attribute_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE OPERATOR CLASS vector_integer_ops
+	DEFAULT FOR TYPE integer USING hnsw AS
+	OPERATOR 2 = (integer, integer),
+	FUNCTION 4 hnsw_attribute_distance(integer, integer);

sql/vector.sql

@@ -266,18 +266,12 @@ COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
 CREATE FUNCTION ivfflat_halfvec_support(internal) RETURNS internal
 	AS 'MODULE_PATHNAME' LANGUAGE C;
 
-CREATE FUNCTION ivfflat_minivec_support(internal) RETURNS internal
-	AS 'MODULE_PATHNAME' LANGUAGE C;
-
 CREATE FUNCTION ivfflat_bit_support(internal) RETURNS internal
 	AS 'MODULE_PATHNAME' LANGUAGE C;
 
 CREATE FUNCTION hnsw_halfvec_support(internal) RETURNS internal
 	AS 'MODULE_PATHNAME' LANGUAGE C;
 
-CREATE FUNCTION hnsw_minivec_support(internal) RETURNS internal
-	AS 'MODULE_PATHNAME' LANGUAGE C;
-
 CREATE FUNCTION hnsw_bit_support(internal) RETURNS internal
 	AS 'MODULE_PATHNAME' LANGUAGE C;
 
@@ -653,295 +647,6 @@ CREATE OPERATOR CLASS halfvec_l1_ops
 	FUNCTION 1 l1_distance(halfvec, halfvec),
 	FUNCTION 3 hnsw_halfvec_support(internal);
 
--- minivec type
-
-CREATE TYPE minivec;
-
-CREATE FUNCTION minivec_in(cstring, oid, integer) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_out(minivec) RETURNS cstring
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_typmod_in(cstring[]) RETURNS integer
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_recv(internal, oid, integer) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_send(minivec) RETURNS bytea
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE TYPE minivec (
-	INPUT     = minivec_in,
-	OUTPUT    = minivec_out,
-	TYPMOD_IN = minivec_typmod_in,
-	RECEIVE   = minivec_recv,
-	SEND      = minivec_send,
-	STORAGE   = external
-);
-
--- minivec functions
-
-CREATE FUNCTION l2_distance(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME', 'minivec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION inner_product(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME', 'minivec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION cosine_distance(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME', 'minivec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION l1_distance(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME', 'minivec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION vector_dims(minivec) RETURNS integer
-	AS 'MODULE_PATHNAME', 'minivec_vector_dims' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION l2_norm(minivec) RETURNS float8
-	AS 'MODULE_PATHNAME', 'minivec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION l2_normalize(minivec) RETURNS minivec
-	AS 'MODULE_PATHNAME', 'minivec_l2_normalize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION binary_quantize(minivec) RETURNS bit
-	AS 'MODULE_PATHNAME', 'minivec_binary_quantize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION subvector(minivec, int, int) RETURNS minivec
-	AS 'MODULE_PATHNAME', 'minivec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
--- minivec private functions
-
-CREATE FUNCTION minivec_add(minivec, minivec) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_sub(minivec, minivec) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_mul(minivec, minivec) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_concat(minivec, minivec) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_lt(minivec, minivec) RETURNS bool
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_le(minivec, minivec) RETURNS bool
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_eq(minivec, minivec) RETURNS bool
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_ne(minivec, minivec) RETURNS bool
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_ge(minivec, minivec) RETURNS bool
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_gt(minivec, minivec) RETURNS bool
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_cmp(minivec, minivec) RETURNS int4
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_l2_squared_distance(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_negative_inner_product(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_spherical_distance(minivec, minivec) RETURNS float8
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
--- minivec cast functions
-
-CREATE FUNCTION minivec(minivec, integer, boolean) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_to_vector(minivec, integer, boolean) RETURNS vector
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION vector_to_minivec(vector, integer, boolean) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION array_to_minivec(integer[], integer, boolean) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION array_to_minivec(real[], integer, boolean) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION array_to_minivec(double precision[], integer, boolean) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION array_to_minivec(numeric[], integer, boolean) RETURNS minivec
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
-CREATE FUNCTION minivec_to_float4(minivec, integer, boolean) RETURNS real[]
-	AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-
--- minivec casts
-
-CREATE CAST (minivec AS minivec)
-	WITH FUNCTION minivec(minivec, integer, boolean) AS IMPLICIT;
-
-CREATE CAST (minivec AS vector)
-	WITH FUNCTION minivec_to_vector(minivec, integer, boolean) AS ASSIGNMENT;
-
-CREATE CAST (vector AS minivec)
-	WITH FUNCTION vector_to_minivec(vector, integer, boolean) AS IMPLICIT;
-
-CREATE CAST (minivec AS real[])
-	WITH FUNCTION minivec_to_float4(minivec, integer, boolean) AS ASSIGNMENT;
-
-CREATE CAST (integer[] AS minivec)
-	WITH FUNCTION array_to_minivec(integer[], integer, boolean) AS ASSIGNMENT;
-
-CREATE CAST (real[] AS minivec)
-	WITH FUNCTION array_to_minivec(real[], integer, boolean) AS ASSIGNMENT;
-
-CREATE CAST (double precision[] AS minivec)
-	WITH FUNCTION array_to_minivec(double precision[], integer, boolean) AS ASSIGNMENT;
-
-CREATE CAST (numeric[] AS minivec)
-	WITH FUNCTION array_to_minivec(numeric[], integer, boolean) AS ASSIGNMENT;
-
--- minivec operators
-
-CREATE OPERATOR <-> (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = l2_distance,
-	COMMUTATOR = '<->'
-);
-
-CREATE OPERATOR <#> (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_negative_inner_product,
-	COMMUTATOR = '<#>'
-);
-
-CREATE OPERATOR <=> (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = cosine_distance,
-	COMMUTATOR = '<=>'
-);
-
-CREATE OPERATOR <+> (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = l1_distance,
-	COMMUTATOR = '<+>'
-);
-
-CREATE OPERATOR + (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_add,
-	COMMUTATOR = +
-);
-
-CREATE OPERATOR - (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_sub
-);
-
-CREATE OPERATOR * (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_mul,
-	COMMUTATOR = *
-);
-
-CREATE OPERATOR || (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_concat
-);
-
-CREATE OPERATOR < (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_lt,
-	COMMUTATOR = > , NEGATOR = >= ,
-	RESTRICT = scalarltsel, JOIN = scalarltjoinsel
-);
-
-CREATE OPERATOR <= (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_le,
-	COMMUTATOR = >= , NEGATOR = > ,
-	RESTRICT = scalarlesel, JOIN = scalarlejoinsel
-);
-
-CREATE OPERATOR = (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_eq,
-	COMMUTATOR = = , NEGATOR = <> ,
-	RESTRICT = eqsel, JOIN = eqjoinsel
-);
-
-CREATE OPERATOR <> (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_ne,
-	COMMUTATOR = <> , NEGATOR = = ,
-	RESTRICT = eqsel, JOIN = eqjoinsel
-);
-
-CREATE OPERATOR >= (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_ge,
-	COMMUTATOR = <= , NEGATOR = < ,
-	RESTRICT = scalargesel, JOIN = scalargejoinsel
-);
-
-CREATE OPERATOR > (
-	LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_gt,
-	COMMUTATOR = < , NEGATOR = <= ,
-	RESTRICT = scalargtsel, JOIN = scalargtjoinsel
-);
-
--- minivec op classes
-
-CREATE OPERATOR CLASS minivec_ops
-	DEFAULT FOR TYPE minivec USING btree AS
-	OPERATOR 1 < ,
-	OPERATOR 2 <= ,
-	OPERATOR 3 = ,
-	OPERATOR 4 >= ,
-	OPERATOR 5 > ,
-	FUNCTION 1 minivec_cmp(minivec, minivec);
-
-CREATE OPERATOR CLASS minivec_l2_ops
-	FOR TYPE minivec USING ivfflat AS
-	OPERATOR 1 <-> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 minivec_l2_squared_distance(minivec, minivec),
-	FUNCTION 3 l2_distance(minivec, minivec),
-	FUNCTION 5 ivfflat_minivec_support(internal);
-
-CREATE OPERATOR CLASS minivec_ip_ops
-	FOR TYPE minivec USING ivfflat AS
-	OPERATOR 1 <#> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 minivec_negative_inner_product(minivec, minivec),
-	FUNCTION 3 minivec_spherical_distance(minivec, minivec),
-	FUNCTION 4 l2_norm(minivec),
-	FUNCTION 5 ivfflat_minivec_support(internal);
-
-CREATE OPERATOR CLASS minivec_cosine_ops
-	FOR TYPE minivec USING ivfflat AS
-	OPERATOR 1 <=> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 cosine_distance(minivec, minivec),
-	FUNCTION 2 l2_norm(minivec),
-	FUNCTION 3 minivec_spherical_distance(minivec, minivec),
-	FUNCTION 4 l2_norm(minivec),
-	FUNCTION 5 ivfflat_minivec_support(internal);
-
-CREATE OPERATOR CLASS minivec_l2_ops
-	FOR TYPE minivec USING hnsw AS
-	OPERATOR 1 <-> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 minivec_l2_squared_distance(minivec, minivec),
-	FUNCTION 3 hnsw_minivec_support(internal);
-
-CREATE OPERATOR CLASS minivec_ip_ops
-	FOR TYPE minivec USING hnsw AS
-	OPERATOR 1 <#> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 minivec_negative_inner_product(minivec, minivec),
-	FUNCTION 3 hnsw_minivec_support(internal);
-
-CREATE OPERATOR CLASS minivec_cosine_ops
-	FOR TYPE minivec USING hnsw AS
-	OPERATOR 1 <=> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 cosine_distance(minivec, minivec),
-	FUNCTION 2 l2_norm(minivec),
-	FUNCTION 3 hnsw_minivec_support(internal);
-
-CREATE OPERATOR CLASS minivec_l1_ops
-	FOR TYPE minivec USING hnsw AS
-	OPERATOR 1 <+> (minivec, minivec) FOR ORDER BY float_ops,
-	FUNCTION 1 l1_distance(minivec, minivec),
-	FUNCTION 3 hnsw_minivec_support(internal);
-
 -- bit functions
 
 CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
@@ -1211,3 +916,13 @@ CREATE OPERATOR CLASS sparsevec_l1_ops
 	OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
 	FUNCTION 1 l1_distance(sparsevec, sparsevec),
 	FUNCTION 3 hnsw_sparsevec_support(internal);
+
+-- hnsw attributes
+
+CREATE FUNCTION hnsw_attribute_distance(integer, integer) RETURNS float8
+	AS 'MODULE_PATHNAME', 'hnsw_int4_attribute_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE OPERATOR CLASS vector_integer_ops
+	DEFAULT FOR TYPE integer USING hnsw AS
+	OPERATOR 2 = (integer, integer),
+	FUNCTION 4 hnsw_attribute_distance(integer, integer);

src/hnsw.c

@@ -12,6 +12,7 @@
 #include "utils/float.h"
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
+#include "utils/spccache.h"
 
 #if PG_VERSION_NUM < 150000
 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -99,7 +100,9 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 {
 	GenericCosts costs;
 	int			m;
-	int			entryLevel;
+	double		ratio;
+	double		startupPages;
+	double		spc_seq_page_cost;
 	Relation	index;
 
 	/* Never use index without order */
@@ -115,21 +118,71 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 
 	MemSet(&costs, 0, sizeof(costs));
 
+	genericcostestimate(root, path, loop_count, &costs);
+
 	index = index_open(path->indexinfo->indexoid, NoLock);
 	HnswGetMetaPageInfo(index, &m, NULL);
 	index_close(index, NoLock);
 
-	/* Approximate entry level */
-	entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
+	/*
+	 * HNSW cost estimation follows a formula that accounts for the total
+	 * number of tuples indexed combined with the parameters that most
+	 * influence the duration of the index scan, namely: m - the number of
+	 * tuples that are scanned in each step of the HNSW graph traversal
+	 * ef_search - which influences the total number of steps taken at layer 0
+	 *
+	 * The source of the vector data can impact how many steps it takes to
+	 * converge on the set of vectors to return to the executor. Currently, we
+	 * use a hardcoded scaling factor (HNSWScanScalingFactor) to help
+	 * influence that, but this could later become a configurable parameter
+	 * based on the cost estimations.
+	 *
+	 * The tuple estimator formula is below:
+	 *
+	 * numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
+	 *
+	 * "entryLevel * m" represents the floor of tuples we need to scan to get
+	 * to layer 0 (L0).
+	 *
+	 * "layer0TuplesMax" is the estimated total number of tuples we'd scan at
+	 * L0 if we weren't discarding already visited tuples as part of the scan.
+	 *
+	 * "layer0Selectivity" estimates the percentage of tuples that are scanned
+	 * at L0, accounting for previously visited tuples, multiplied by the
+	 * "scalingFactor" (currently hardcoded).
+	 */
+	if (path->indexinfo->tuples > 0)
+	{
+		double		scalingFactor = 0.55;
+		int			entryLevel = (int) (log(path->indexinfo->tuples) * HnswGetMl(m));
+		int			layer0TuplesMax = HnswGetLayerM(m, 0) * hnsw_ef_search;
+		double		layer0Selectivity = scalingFactor * log(path->indexinfo->tuples) / (log(m) * (1 + log(hnsw_ef_search)));
 
-	/* TODO Improve estimate of visited tuples (currently underestimates) */
-	/* Account for number of tuples (or entry level), m, and ef_search */
-	costs.numIndexTuples = (entryLevel + 2) * m;
+		ratio = (entryLevel * m + layer0TuplesMax * layer0Selectivity) / path->indexinfo->tuples;
 
-	genericcostestimate(root, path, loop_count, &costs);
+		if (ratio > 1)
+			ratio = 1;
+	}
+	else
+		ratio = 1;
 
-	/* Use total cost since most work happens before first tuple is returned */
-	*indexStartupCost = costs.indexTotalCost;
+	get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
+
+	/* Startup cost is cost before returning the first row */
+	costs.indexStartupCost = costs.indexTotalCost * ratio;
+
+	/* Adjust cost if needed since TOAST not included in seq scan cost */
+	startupPages = costs.numIndexPages * ratio;
+	if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
+	{
+		/* Change all page cost from random to sequential */
+		costs.indexStartupCost -= startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+
+		/* Remove cost of extra pages */
+		costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
+	}
+
+	*indexStartupCost = costs.indexStartupCost;
 	*indexTotalCost = costs.indexTotalCost;
 	*indexSelectivity = costs.indexSelectivity;
 	*indexCorrelation = costs.indexCorrelation;
@@ -174,13 +227,13 @@ hnswhandler(PG_FUNCTION_ARGS)
 	IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
 
 	amroutine->amstrategies = 0;
-	amroutine->amsupport = 3;
+	amroutine->amsupport = 4;
 	amroutine->amoptsprocnum = 0;
 	amroutine->amcanorder = false;
 	amroutine->amcanorderbyop = true;
 	amroutine->amcanbackward = false;	/* can change direction mid-scan */
 	amroutine->amcanunique = false;
-	amroutine->amcanmulticol = false;
+	amroutine->amcanmulticol = true;
 	amroutine->amoptionalkey = true;
 	amroutine->amsearcharray = false;
 	amroutine->amsearchnulls = false;
@@ -232,3 +285,17 @@ hnswhandler(PG_FUNCTION_ARGS)
 
 	PG_RETURN_POINTER(amroutine);
 }
+
+/*
+ * Get the distance between two int4 attributes
+ */
+PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_int4_attribute_distance);
+Datum
+hnsw_int4_attribute_distance(PG_FUNCTION_ARGS)
+{
+	int32		a = PG_GETARG_INT32(0);
+	int32		b = PG_GETARG_INT32(1);
+	double		distance = ((double) a) - ((double) b);
+
+	PG_RETURN_FLOAT8(distance);
+}

src/hnsw.h

@@ -19,6 +19,7 @@
 #define HNSW_DISTANCE_PROC 1
 #define HNSW_NORM_PROC 2
 #define HNSW_TYPE_INFO_PROC 3
+#define HNSW_ATTRIBUTE_DISTANCE_PROC 4
 
 #define HNSW_VERSION	1
 #define HNSW_MAGIC_NUMBER 0xA953A953
@@ -104,6 +105,8 @@
 #define HnswPtrPointer(hp) (hp).ptr
 #define HnswPtrOffset(hp) relptr_offset((hp).relptr)
 
+#define HnswUseIndexTuple(index) (IndexRelationGetNumberOfAttributes(index) > 1)
+
 /* Variables */
 extern int	hnsw_ef_search;
 extern int	hnsw_lock_tranche_id;
@@ -121,6 +124,7 @@ HnswPtrDeclare(HnswElementData, HnswElementRelptr, HnswElementPtr);
 HnswPtrDeclare(HnswNeighborArray, HnswNeighborArrayRelptr, HnswNeighborArrayPtr);
 HnswPtrDeclare(HnswNeighborArrayPtr, HnswNeighborsRelptr, HnswNeighborsPtr);
 HnswPtrDeclare(char, DatumRelptr, DatumPtr);
+HnswPtrDeclare(IndexTupleData, IndexTupleRelptr, IndexTuplePtr);
 
 struct HnswElementData
 {
@@ -136,6 +140,7 @@ struct HnswElementData
 	OffsetNumber neighborOffno;
 	BlockNumber neighborPage;
 	DatumPtr	value;
+	IndexTuplePtr itup;
 	LWLock		lock;
 };
 
@@ -160,7 +165,8 @@ typedef struct HnswSearchCandidate
 	pairingheap_node c_node;
 	pairingheap_node w_node;
 	HnswElementPtr element;
-	float		distance;
+	double		distance;
+	bool		matches;
 }			HnswSearchCandidate;
 
 /* HNSW index options */
@@ -185,8 +191,8 @@ typedef struct HnswGraph
 
 	/* Allocations state */
 	LWLock		allocatorLock;
-	long		memoryUsed;
-	long		memoryTotal;
+	Size		memoryUsed;
+	Size		memoryTotal;
 
 	/* Flushed state */
 	LWLock		flushLock;
@@ -256,15 +262,17 @@ typedef struct HnswBuildState
 	double		reltuples;
 
 	/* Support functions */
-	FmgrInfo   *procinfo;
+	FmgrInfo   *procinfo[2];
 	FmgrInfo   *normprocinfo;
-	Oid			collation;
+	Oid		   *collation;
 
 	/* Variables */
 	HnswGraph	graphData;
 	HnswGraph  *graph;
 	double		ml;
 	int			maxLevel;
+	bool		useIndexTuple;
+	TupleDesc	tupdesc;
 
 	/* Memory */
 	MemoryContext graphCtx;
@@ -333,9 +341,9 @@ typedef struct HnswScanOpaqueData
 	MemoryContext tmpCtx;
 
 	/* Support functions */
-	FmgrInfo   *procinfo;
+	FmgrInfo   *procinfo[2];
 	FmgrInfo   *normprocinfo;
-	Oid			collation;
+	Oid		   *collation;
 }			HnswScanOpaqueData;
 
 typedef HnswScanOpaqueData * HnswScanOpaque;
@@ -353,8 +361,8 @@ typedef struct HnswVacuumState
 	int			efConstruction;
 
 	/* Support functions */
-	FmgrInfo   *procinfo;
-	Oid			collation;
+	FmgrInfo   *procinfo[2];
+	Oid		   *collation;
 
 	/* Variables */
 	struct tidhash_hash *deleted;
@@ -375,28 +383,32 @@ bool		HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
 Buffer		HnswNewBuffer(Relation index, ForkNumber forkNum);
 void		HnswInitPage(Buffer buf, Page page);
 void		HnswInit(void);
-List	   *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
+List	   *HnswSearchLayer(char *base, Datum q, IndexTuple qtup, ScanKeyData *keyData, List *ep, int ef, int lc, Relation index, FmgrInfo **procinfo, Oid *collation, int m, bool inserting, HnswElement skipElement, bool inMemory);
 HnswElement HnswGetEntryPoint(Relation index);
 void		HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
 void	   *HnswAlloc(HnswAllocator * allocator, Size size);
 HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
 HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
-void		HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
-HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
+void		HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo **procinfo, Oid *collation, int m, int efConstruction, bool existing, bool inMemory);
+HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, IndexTuple qtup, ScanKeyData *keyData, Relation rel, FmgrInfo **procinfo, Oid *collation, bool loadVec, bool inMemory);
 void		HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
 void		HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
 void		HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
+HnswNeighborArray *HnswInitNeighborArray(int lm, HnswAllocator * allocator);
 void		HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * alloc);
 bool		HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
-void		HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
-void		HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
-void		HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance);
-void		HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
-void		HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
-void		HnswLoadNeighbors(HnswElement element, Relation index, int m);
+void		HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement e, int m, bool checkExisting, bool building);
+void		HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec, Relation index);
+void		HnswLoadElement(HnswElement element, double *distance, bool *matches, Datum *q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation, bool loadVec, double *maxDistance);
+void		HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element, bool useIndexTuple);
+void		HnswUpdateConnection(char *base, HnswNeighborArray * neighbors, HnswElement newElement, float distance, int lm, int *updateIdx, Relation index, FmgrInfo **procinfo, Oid *collation);
+bool		HnswLoadNeighborTids(HnswElement element, ItemPointerData *indextids, Relation index, int m, int lm, int lc);
 void		HnswInitLockTranche(void);
 const		HnswTypeInfo *HnswGetTypeInfo(Relation index);
 PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
+void		HnswInitProcinfo(FmgrInfo **procinfo, Oid **collation, Relation index);
+Size		HnswGetElementTupleSize(char *base, HnswElement element, bool useIndexTuple);
+bool		HnswIndexTupleIsEqual(IndexTuple a, IndexTuple b, TupleDesc tupdesc);
 
 /* Index access methods */
 IndexBuildResult *hnswbuild(Relation heap, Relation index, IndexInfo *indexInfo);

src/hnswbuild.c

@@ -148,6 +148,7 @@ CreateGraphPages(HnswBuildState * buildstate)
 	Page		page;
 	HnswElementPtr iter = buildstate->graph->head;
 	char	   *base = buildstate->hnswarea;
+	bool		useIndexTuple = buildstate->useIndexTuple;
 
 	/* Calculate sizes */
 	maxSize = HNSW_MAX_SIZE;
@@ -167,7 +168,6 @@ CreateGraphPages(HnswBuildState * buildstate)
 		Size		etupSize;
 		Size		ntupSize;
 		Size		combinedSize;
-		Pointer		valuePtr = HnswPtrAccess(base, element->value);
 
 		/* Update iterator */
 		iter = element->next;
@@ -176,7 +176,7 @@ CreateGraphPages(HnswBuildState * buildstate)
 		MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE);
 
 		/* Calculate sizes */
-		etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(valuePtr));
+		etupSize = HnswGetElementTupleSize(base, element, useIndexTuple);
 		ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, buildstate->m);
 		combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
 
@@ -186,7 +186,7 @@ CreateGraphPages(HnswBuildState * buildstate)
 					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 					 errmsg("index tuple too large")));
 
-		HnswSetElementTuple(base, etup, element);
+		HnswSetElementTuple(base, etup, element, useIndexTuple);
 
 		/* Keep element and neighbors on the same page if possible */
 		if (PageGetFreeSpace(page) < etupSize || (combinedSize <= maxSize && PageGetFreeSpace(page) < combinedSize))
@@ -327,20 +327,29 @@ AddDuplicateInMemory(HnswElement element, HnswElement dup)
  * Find duplicate element
  */
 static bool
-FindDuplicateInMemory(char *base, HnswElement element)
+FindDuplicateInMemory(char *base, HnswElement element, bool useIndexTuple, TupleDesc tupdesc)
 {
 	HnswNeighborArray *neighbors = HnswGetNeighbors(base, element, 0);
 	Datum		value = HnswGetValue(base, element);
+	IndexTuple	itup = HnswPtrAccess(base, element->itup);
 
 	for (int i = 0; i < neighbors->length; i++)
 	{
 		HnswCandidate *neighbor = &neighbors->items[i];
 		HnswElement neighborElement = HnswPtrAccess(base, neighbor->element);
-		Datum		neighborValue = HnswGetValue(base, neighborElement);
 
-		/* Exit early since ordered by distance */
-		if (!datumIsEqual(value, neighborValue, false, -1))
-			return false;
+		if (useIndexTuple)
+		{
+			/* Exit early since ordered by distance */
+			if (!HnswIndexTupleIsEqual(itup, HnswPtrAccess(base, neighborElement->itup), tupdesc))
+				return false;
+		}
+		else
+		{
+			/* Exit early since ordered by distance */
+			if (!datumIsEqual(value, HnswGetValue(base, neighborElement), false, -1))
+				return false;
+		}
 
 		/* Check for space */
 		if (AddDuplicateInMemory(element, neighborElement))
@@ -366,7 +375,7 @@ AddElementInMemory(char *base, HnswGraph * graph, HnswElement element)
  * Update neighbors
  */
 static void
-UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswElement e, int m)
+UpdateNeighborsInMemory(char *base, Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement e, int m)
 {
 	for (int lc = e->level; lc >= 0; lc--)
 	{
@@ -388,7 +397,7 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
 			Assert(neighborElement);
 
 			LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
-			HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
+			HnswUpdateConnection(base, HnswGetNeighbors(base, neighborElement, lc), e, hc->distance, lm, NULL, index, procinfo, collation);
 			LWLockRelease(&neighborElement->lock);
 		}
 	}
@@ -398,20 +407,20 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
  * Update graph in memory
  */
 static void
-UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
+UpdateGraphInMemory(FmgrInfo **procinfo, Oid *collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
 {
 	HnswGraph  *graph = buildstate->graph;
 	char	   *base = buildstate->hnswarea;
 
 	/* Look for duplicate */
-	if (FindDuplicateInMemory(base, element))
+	if (FindDuplicateInMemory(base, element, buildstate->useIndexTuple, buildstate->tupdesc))
 		return;
 
 	/* Add element */
 	AddElementInMemory(base, graph, element);
 
 	/* Update neighbors */
-	UpdateNeighborsInMemory(base, procinfo, collation, element, m);
+	UpdateNeighborsInMemory(base, buildstate->index, procinfo, collation, element, m);
 
 	/* Update entry point if needed (already have lock) */
 	if (entryPoint == NULL || element->level > entryPoint->level)
@@ -424,8 +433,9 @@ UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int
 static void
 InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
 {
-	FmgrInfo   *procinfo = buildstate->procinfo;
-	Oid			collation = buildstate->collation;
+	Relation	index = buildstate->index;
+	FmgrInfo  **procinfo = buildstate->procinfo;
+	Oid		   *collation = buildstate->collation;
 	HnswGraph  *graph = buildstate->graph;
 	HnswElement entryPoint;
 	LWLock	   *entryLock = &graph->entryLock;
@@ -458,7 +468,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
 	}
 
 	/* Find neighbors for element */
-	HnswFindElementNeighbors(base, element, entryPoint, NULL, procinfo, collation, m, efConstruction, false);
+	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false, true);
 
 	/* Update graph in memory */
 	UpdateGraphInMemory(procinfo, collation, element, m, efConstruction, entryPoint, buildstate);
@@ -481,6 +491,11 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	Pointer		valuePtr;
 	LWLock	   *flushLock = &graph->flushLock;
 	char	   *base = buildstate->hnswarea;
+	bool		useIndexTuple = buildstate->useIndexTuple;
+	TupleDesc	tupdesc = buildstate->tupdesc;
+	IndexTuple	itup;
+	Size		itupSize;
+	IndexTuple	itupPtr;
 
 	/* Detoast once for all calls */
 	Datum		value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
@@ -492,10 +507,10 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	/* Normalize if needed */
 	if (buildstate->normprocinfo != NULL)
 	{
-		if (!HnswCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
+		if (!HnswCheckNorm(buildstate->normprocinfo, buildstate->collation[0], value))
 			return false;
 
-		value = HnswNormValue(typeInfo, buildstate->collation, value);
+		value = HnswNormValue(typeInfo, buildstate->collation[0], value);
 	}
 
 	/* Get datum size */
@@ -546,7 +561,17 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 
 	/* Ok, we can proceed to allocate the element */
 	element = HnswInitElement(base, heaptid, buildstate->m, buildstate->ml, buildstate->maxLevel, allocator);
-	valuePtr = HnswAlloc(allocator, valueSize);
+
+	if (useIndexTuple)
+	{
+		/* TODO fix */
+		values[0] = value;
+		itup = index_form_tuple(tupdesc, values, isnull);
+		itupSize = IndexTupleSize(itup);
+		itupPtr = HnswAlloc(allocator, itupSize);
+	}
+	else
+		valuePtr = HnswAlloc(allocator, valueSize);
 
 	/*
 	 * We have now allocated the space needed for the element, so we don't
@@ -556,8 +581,19 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	LWLockRelease(&graph->allocatorLock);
 
 	/* Copy the datum */
-	memcpy(valuePtr, DatumGetPointer(value), valueSize);
-	HnswPtrStore(base, element->value, valuePtr);
+	if (useIndexTuple)
+	{
+		bool		unused;
+
+		memcpy(itupPtr, itup, itupSize);
+		HnswPtrStore(base, element->itup, itupPtr);
+		HnswPtrStore(base, element->value, DatumGetPointer(index_getattr(itupPtr, 1, tupdesc, &unused)));
+	}
+	else
+	{
+		memcpy(valuePtr, DatumGetPointer(value), valueSize);
+		HnswPtrStore(base, element->value, valuePtr);
+	}
 
 	/* Create a lock for the element */
 	LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -607,7 +643,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
  * Initialize the graph
  */
 static void
-InitGraph(HnswGraph * graph, char *base, long memoryTotal)
+InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
 {
 	/* Initialize the lock tranche if needed */
 	HnswInitLockTranche();
@@ -684,6 +720,19 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("type not supported for hnsw index")));
 
+	/* TODO See if needed */
+	if (IndexRelationGetNumberOfKeyAttributes(index) > 2)
+		elog(ERROR, "index cannot have more than two columns");
+
+	if (!OidIsValid(index_getprocid(index, 1, HNSW_DISTANCE_PROC)))
+		elog(ERROR, "first column must be a vector");
+
+	for (int i = 1; i < IndexRelationGetNumberOfKeyAttributes(index); i++)
+	{
+		if (!OidIsValid(index_getprocid(index, i + 1, HNSW_ATTRIBUTE_DISTANCE_PROC)))
+			elog(ERROR, "column %d cannot be a vector", i + 1);
+	}
+
 	/* Require column to have dimensions to be indexed */
 	if (buildstate->dimensions < 0)
 		ereport(ERROR,
@@ -704,14 +753,15 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
 	buildstate->indtuples = 0;
 
 	/* Get support functions */
-	buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+	HnswInitProcinfo(buildstate->procinfo, &buildstate->collation, index);
 	buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
-	buildstate->collation = index->rd_indcollation[0];
 
-	InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * 1024L);
+	InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L);
 	buildstate->graph = &buildstate->graphData;
 	buildstate->ml = HnswGetMl(buildstate->m);
 	buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
+	buildstate->useIndexTuple = HnswUseIndexTuple(index);
+	buildstate->tupdesc = RelationGetDescr(index);
 
 	buildstate->graphCtx = GenerationContextCreate(CurrentMemoryContext,
 												   "Hnsw build graph context",

src/hnswinsert.c

@@ -154,9 +154,10 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 	OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
 	BlockNumber newInsertPage = InvalidBlockNumber;
 	char	   *base = NULL;
+	bool		useIndexTuple = HnswUseIndexTuple(index);
 
 	/* Calculate sizes */
-	etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(HnswPtrAccess(base, e->value)));
+	etupSize = HnswGetElementTupleSize(base, e, useIndexTuple);
 	ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m);
 	combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
 	maxSize = HNSW_MAX_SIZE;
@@ -164,7 +165,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 
 	/* Prepare element tuple */
 	etup = palloc0(etupSize);
-	HnswSetElementTuple(base, etup, e);
+	HnswSetElementTuple(base, etup, e, useIndexTuple);
 
 	/* Prepare neighbor tuple */
 	ntup = palloc0(ntupSize);
@@ -334,6 +335,107 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 		*updatedInsertPage = newInsertPage;
 }
 
+/*
+ * Load neighbors
+ */
+static HnswNeighborArray *
+HnswLoadNeighbors(HnswElement element, Relation index, int m, int lm, int lc)
+{
+	char	   *base = NULL;
+	HnswNeighborArray *neighbors = HnswInitNeighborArray(lm, NULL);
+	ItemPointerData indextids[HNSW_MAX_M * 2];
+
+	if (!HnswLoadNeighborTids(element, indextids, index, m, lm, lc))
+		return neighbors;
+
+	for (int i = 0; i < lm; i++)
+	{
+		ItemPointer indextid = &indextids[i];
+		HnswElement e;
+		HnswCandidate *hc;
+
+		if (!ItemPointerIsValid(indextid))
+			break;
+
+		e = HnswInitElementFromBlock(ItemPointerGetBlockNumber(indextid), ItemPointerGetOffsetNumber(indextid));
+		hc = &neighbors->items[neighbors->length++];
+		HnswPtrStore(base, hc->element, e);
+	}
+
+	return neighbors;
+}
+
+/*
+ * Load elements for insert
+ */
+static void
+LoadElementsForInsert(HnswNeighborArray * neighbors, Datum q, IndexTuple qtup, int *idx, Relation index, FmgrInfo **procinfo, Oid *collation)
+{
+	char	   *base = NULL;
+
+	for (int i = 0; i < neighbors->length; i++)
+	{
+		HnswCandidate *hc = &neighbors->items[i];
+		HnswElement element = HnswPtrAccess(base, hc->element);
+		double		distance;
+		bool		matches;
+
+		HnswLoadElement(element, &distance, &matches, &q, qtup, NULL, index, procinfo, collation, true, NULL);
+		hc->distance = distance;
+
+		/* Prune element if being deleted */
+		if (element->heaptidsLength == 0)
+		{
+			*idx = i;
+			break;
+		}
+	}
+}
+
+/*
+ * Get update index
+ */
+static int
+GetUpdateIndex(HnswElement element, HnswElement newElement, float distance, int m, int lm, int lc, Relation index, FmgrInfo **procinfo, Oid *collation, MemoryContext updateCtx)
+{
+	char	   *base = NULL;
+	int			idx = -1;
+	HnswNeighborArray *neighbors;
+	MemoryContext oldCtx = MemoryContextSwitchTo(updateCtx);
+
+	/*
+	 * Get latest neighbors since they may have changed. Do not lock yet since
+	 * selecting neighbors can take time. Could use optimistic locking to
+	 * retry if another update occurs before getting exclusive lock.
+	 */
+	neighbors = HnswLoadNeighbors(element, index, m, lm, lc);
+
+	/*
+	 * Could improve performance for vacuuming by checking neighbors against
+	 * list of elements being deleted to find index. It's important to exclude
+	 * already deleted elements for this since they can be replaced at any
+	 * time.
+	 */
+
+	if (neighbors->length < lm)
+		idx = -2;
+	else
+	{
+		Datum		q = HnswGetValue(base, element);
+		IndexTuple	qtup = HnswPtrAccess(base, element->itup);;
+
+		LoadElementsForInsert(neighbors, q, qtup, &idx, index, procinfo, collation);
+
+		if (idx == -1)
+			HnswUpdateConnection(base, neighbors, newElement, distance, lm, &idx, index, procinfo, collation);
+	}
+
+	MemoryContextSwitchTo(oldCtx);
+	MemoryContextReset(updateCtx);
+
+	return idx;
+}
+
 /*
  * Check if connection already exists
  */
@@ -354,14 +456,94 @@ ConnectionExists(HnswElement e, HnswNeighborTuple ntup, int startIdx, int lm)
 	return false;
 }
 
+/*
+ * Update neighbor
+ */
+static void
+UpdateNeighborOnDisk(HnswElement element, HnswElement newElement, int idx, int m, int lm, int lc, Relation index, bool checkExisting, bool building)
+{
+	Buffer		buf;
+	Page		page;
+	GenericXLogState *state;
+	HnswNeighborTuple ntup;
+	int			startIdx;
+	OffsetNumber offno = element->neighborOffno;
+
+	/* Register page */
+	buf = ReadBuffer(index, element->neighborPage);
+	LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
+	if (building)
+	{
+		state = NULL;
+		page = BufferGetPage(buf);
+	}
+	else
+	{
+		state = GenericXLogStart(index);
+		page = GenericXLogRegisterBuffer(state, buf, 0);
+	}
+
+	/* Get tuple */
+	ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
+
+	/* Calculate index for update */
+	startIdx = (element->level - lc) * m;
+
+	/* Check for existing connection */
+	if (checkExisting && ConnectionExists(newElement, ntup, startIdx, lm))
+		idx = -1;
+	else if (idx == -2)
+	{
+		/* Find free offset if still exists */
+		/* TODO Retry updating connections if not */
+		for (int j = 0; j < lm; j++)
+		{
+			if (!ItemPointerIsValid(&ntup->indextids[startIdx + j]))
+			{
+				idx = startIdx + j;
+				break;
+			}
+		}
+	}
+	else
+		idx += startIdx;
+
+	/* Make robust to issues */
+	if (idx >= 0 && idx < ntup->count)
+	{
+		ItemPointer indextid = &ntup->indextids[idx];
+
+		/* Update neighbor on the buffer */
+		ItemPointerSet(indextid, newElement->blkno, newElement->offno);
+
+		/* Commit */
+		if (building)
+			MarkBufferDirty(buf);
+		else
+			GenericXLogFinish(state);
+	}
+	else if (!building)
+		GenericXLogAbort(state);
+
+	UnlockReleaseBuffer(buf);
+}
+
 /*
  * Update neighbors
  */
 void
-HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building)
+HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement e, int m, bool checkExisting, bool building)
 {
 	char	   *base = NULL;
 
+	/* Use separate memory context to improve performance for larger vectors */
+	MemoryContext updateCtx = GenerationContextCreate(CurrentMemoryContext,
+													  "Hnsw insert update context",
+#if PG_VERSION_NUM >= 150000
+													  128 * 1024, 128 * 1024,
+#endif
+													  128 * 1024);
+
 	for (int lc = e->level; lc >= 0; lc--)
 	{
 		int			lm = HnswGetLayerM(m, lc);
@@ -370,96 +552,20 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
 		for (int i = 0; i < neighbors->length; i++)
 		{
 			HnswCandidate *hc = &neighbors->items[i];
-			Buffer		buf;
-			Page		page;
-			GenericXLogState *state;
-			HnswNeighborTuple ntup;
-			int			idx = -1;
-			int			startIdx;
 			HnswElement neighborElement = HnswPtrAccess(base, hc->element);
-			OffsetNumber offno = neighborElement->neighborOffno;
+			int			idx;
 
-			/*
-			 * Get latest neighbors since they may have changed. Do not lock
-			 * yet since selecting neighbors can take time. Could use
-			 * optimistic locking to retry if another update occurs before
-			 * getting exclusive lock.
-			 */
-			HnswLoadNeighbors(neighborElement, index, m);
-
-			/*
-			 * Could improve performance for vacuuming by checking neighbors
-			 * against list of elements being deleted to find index. It's
-			 * important to exclude already deleted elements for this since
-			 * they can be replaced at any time.
-			 */
-
-			/* Select neighbors */
-			HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, procinfo, collation);
+			idx = GetUpdateIndex(neighborElement, e, hc->distance, m, lm, lc, index, procinfo, collation, updateCtx);
 
 			/* New element was not selected as a neighbor */
 			if (idx == -1)
 				continue;
 
-			/* Register page */
-			buf = ReadBuffer(index, neighborElement->neighborPage);
-			LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
-			if (building)
-			{
-				state = NULL;
-				page = BufferGetPage(buf);
-			}
-			else
-			{
-				state = GenericXLogStart(index);
-				page = GenericXLogRegisterBuffer(state, buf, 0);
-			}
-
-			/* Get tuple */
-			ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
-
-			/* Calculate index for update */
-			startIdx = (neighborElement->level - lc) * m;
-
-			/* Check for existing connection */
-			if (checkExisting && ConnectionExists(e, ntup, startIdx, lm))
-				idx = -1;
-			else if (idx == -2)
-			{
-				/* Find free offset if still exists */
-				/* TODO Retry updating connections if not */
-				for (int j = 0; j < lm; j++)
-				{
-					if (!ItemPointerIsValid(&ntup->indextids[startIdx + j]))
-					{
-						idx = startIdx + j;
-						break;
-					}
-				}
-			}
-			else
-				idx += startIdx;
-
-			/* Make robust to issues */
-			if (idx >= 0 && idx < ntup->count)
-			{
-				ItemPointer indextid = &ntup->indextids[idx];
-
-				/* Update neighbor on the buffer */
-				ItemPointerSet(indextid, e->blkno, e->offno);
-
-				/* Commit */
-				if (building)
-					MarkBufferDirty(buf);
-				else
-					GenericXLogFinish(state);
-			}
-			else if (!building)
-				GenericXLogAbort(state);
-
-			UnlockReleaseBuffer(buf);
+			UpdateNeighborOnDisk(neighborElement, e, idx, m, lm, lc, index, checkExisting, building);
 		}
 	}
+
+	MemoryContextDelete(updateCtx);
 }
 
 /*
@@ -527,16 +633,26 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
 	char	   *base = NULL;
 	HnswNeighborArray *neighbors = HnswGetNeighbors(base, element, 0);
 	Datum		value = HnswGetValue(base, element);
+	IndexTuple	itup = HnswPtrAccess(base, element->itup);
+	TupleDesc	tupdesc = RelationGetDescr(index);
 
 	for (int i = 0; i < neighbors->length; i++)
 	{
 		HnswCandidate *neighbor = &neighbors->items[i];
 		HnswElement neighborElement = HnswPtrAccess(base, neighbor->element);
-		Datum		neighborValue = HnswGetValue(base, neighborElement);
 
-		/* Exit early since ordered by distance */
-		if (!datumIsEqual(value, neighborValue, false, -1))
-			return false;
+		if (HnswUseIndexTuple(index))
+		{
+			/* Exit early since ordered by distance */
+			if (!HnswIndexTupleIsEqual(itup, HnswPtrAccess(base, neighborElement->itup), tupdesc))
+				return false;
+		}
+		else
+		{
+			/* Exit early since ordered by distance */
+			if (!datumIsEqual(value, HnswGetValue(base, neighborElement), false, -1))
+				return false;
+		}
 
 		if (AddDuplicateOnDisk(index, element, neighborElement, building))
 			return true;
@@ -549,7 +665,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
  * Update graph on disk
  */
 static void
-UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
+UpdateGraphOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
 {
 	BlockNumber newInsertPage = InvalidBlockNumber;
 
@@ -582,11 +698,13 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
 	HnswElement element;
 	int			m;
 	int			efConstruction = HnswGetEfConstruction(index);
-	FmgrInfo   *procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
-	Oid			collation = index->rd_indcollation[0];
+	FmgrInfo   *procinfo[2];
+	Oid		   *collation;
 	LOCKMODE	lockmode = ShareLock;
 	char	   *base = NULL;
 
+	HnswInitProcinfo(procinfo, &collation, index);
+
 	/*
 	 * Get a shared lock. This allows vacuum to ensure no in-flight inserts
 	 * before repairing graph. Use a page lock so it does not interfere with
@@ -599,7 +717,23 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
 
 	/* Create an element */
 	element = HnswInitElement(base, heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
-	HnswPtrStore(base, element->value, DatumGetPointer(value));
+	if (HnswUseIndexTuple(index))
+	{
+		/* TODO no toast */
+		TupleDesc	tupdesc = RelationGetDescr(index);
+		IndexTuple	itup;
+		bool		unused;
+
+		/* TODO fix */
+		values[0] = value;
+		itup = index_form_tuple(tupdesc, values, isnull);
+
+		HnswPtrStore(base, element->itup, itup);
+		HnswPtrStore(base, element->value, DatumGetPointer(index_getattr(itup, 1, tupdesc, &unused)));
+
+	}
+	else
+		HnswPtrStore(base, element->value, DatumGetPointer(value));
 
 	/* Prevent concurrent inserts when likely updating entry point */
 	if (entryPoint == NULL || element->level > entryPoint->level)
@@ -616,7 +750,7 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
 	}
 
 	/* Find neighbors for element */
-	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false);
+	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false, false);
 
 	/* Update graph on disk */
 	UpdateGraphOnDisk(index, procinfo, collation, element, m, efConstruction, entryPoint, building);
@@ -636,7 +770,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
 	Datum		value;
 	const		HnswTypeInfo *typeInfo = HnswGetTypeInfo(index);
 	FmgrInfo   *normprocinfo;
-	Oid			collation = index->rd_indcollation[0];
+	Oid		   *collation = index->rd_indcollation;
 
 	/* Detoast once for all calls */
 	value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
@@ -649,10 +783,10 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
 	normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
 	if (normprocinfo != NULL)
 	{
-		if (!HnswCheckNorm(normprocinfo, collation, value))
+		if (!HnswCheckNorm(normprocinfo, collation[0], value))
 			return;
 
-		value = HnswNormValue(typeInfo, collation, value);
+		value = HnswNormValue(typeInfo, collation[0], value);
 	}
 
 	HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false);

src/hnswscan.c

@@ -15,13 +15,15 @@ GetScanItems(IndexScanDesc scan, Datum q)
 {
 	HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
 	Relation	index = scan->indexRelation;
-	FmgrInfo   *procinfo = so->procinfo;
-	Oid			collation = so->collation;
+	FmgrInfo  **procinfo = so->procinfo;
+	Oid		   *collation = so->collation;
 	List	   *ep;
 	List	   *w;
 	int			m;
 	HnswElement entryPoint;
 	char	   *base = NULL;
+	bool		inMemory = false;
+	ScanKeyData *keyData = scan->keyData;
 
 	/* Get m and entry point */
 	HnswGetMetaPageInfo(index, &m, &entryPoint);
@@ -29,15 +31,15 @@ GetScanItems(IndexScanDesc scan, Datum q)
 	if (entryPoint == NULL)
 		return NIL;
 
-	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
+	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, NULL, keyData, index, procinfo, collation, false, inMemory));
 
 	for (int lc = entryPoint->level; lc >= 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
+		w = HnswSearchLayer(base, q, NULL, keyData, ep, 1, lc, index, procinfo, collation, m, false, NULL, inMemory);
 		ep = w;
 	}
 
-	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
+	return HnswSearchLayer(base, q, NULL, keyData, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, inMemory);
 }
 
 /*
@@ -61,7 +63,7 @@ GetScanValue(IndexScanDesc scan)
 
 		/* Normalize if needed */
 		if (so->normprocinfo != NULL)
-			value = HnswNormValue(so->typeInfo, so->collation, value);
+			value = HnswNormValue(so->typeInfo, so->collation[0], value);
 	}
 
 	return value;
@@ -86,9 +88,8 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
 									   ALLOCSET_DEFAULT_SIZES);
 
 	/* Set support functions */
-	so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+	HnswInitProcinfo(so->procinfo, &so->collation, index);
 	so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
-	so->collation = index->rd_indcollation[0];
 
 	scan->opaque = so;
 
@@ -173,7 +174,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 		ItemPointer heaptid;
 
 		/* Move to next element if no valid heap TIDs */
-		if (element->heaptidsLength == 0)
+		if (!hc->matches || element->heaptidsLength == 0)
 		{
 			so->w = list_delete_last(so->w);
 			continue;

src/hnswutils.c

@@ -153,15 +153,50 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
 	return index_getprocinfo(index, 1, procnum);
 }
 
+/*
+ * Init procinfo
+ */
+void
+HnswInitProcinfo(FmgrInfo **procinfo, Oid **collation, Relation index)
+{
+	procinfo[0] = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
+
+	if (IndexRelationGetNumberOfKeyAttributes(index) > 1)
+		procinfo[1] = index_getprocinfo(index, 2, HNSW_ATTRIBUTE_DISTANCE_PROC);
+
+	*collation = index->rd_indcollation;
+}
+
+/*
+ * Get element tuple size
+ */
+Size
+HnswGetElementTupleSize(char *base, HnswElement element, bool useIndexTuple)
+{
+	Size		size;
+
+	if (useIndexTuple)
+	{
+		IndexTuple	itup = HnswPtrAccess(base, element->itup);
+
+		size = IndexTupleSize(itup);
+	}
+	else
+	{
+		Pointer		valuePtr = HnswPtrAccess(base, element->value);
+
+		size = VARSIZE_ANY(valuePtr);
+	}
+
+	return HNSW_ELEMENT_TUPLE_SIZE(size);
+}
+
 /*
  * Normalize value
  */
 Datum
 HnswNormValue(const HnswTypeInfo * typeInfo, Oid collation, Datum value)
 {
-	if (!typeInfo->normalize)
-		return value;
-
 	return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
 }
 
@@ -174,6 +209,37 @@ HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
 	return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
 }
 
+/*
+ * Check if index tuples are equal
+ */
+bool
+HnswIndexTupleIsEqual(IndexTuple a, IndexTuple b, TupleDesc tupdesc)
+{
+	for (int i = 0; i < tupdesc->natts; i++)
+	{
+		bool		nullA;
+		bool		nullB;
+
+		Datum		datumA = index_getattr(a, i + 1, tupdesc, &nullA);
+		Datum		datumB = index_getattr(b, i + 1, tupdesc, &nullB);
+
+		if (nullA || nullB)
+		{
+			if (nullA != nullB)
+				return false;
+		}
+		else
+		{
+			Form_pg_attribute att = TupleDescAttr(tupdesc, i);
+
+			if (!datumIsEqual(datumA, datumB, att->attbyval, att->attlen))
+				return false;
+		}
+	}
+
+	return true;
+}
+
 /*
  * New buffer
  */
@@ -200,7 +266,7 @@ HnswInitPage(Buffer buf, Page page)
 /*
  * Allocate a neighbor array
  */
-static HnswNeighborArray *
+HnswNeighborArray *
 HnswInitNeighborArray(int lm, HnswAllocator * allocator)
 {
 	HnswNeighborArray *a = HnswAlloc(allocator, HNSW_NEIGHBOR_ARRAY_SIZE(lm));
@@ -260,6 +326,7 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
 	HnswInitNeighbors(base, element, m, allocator);
 
 	HnswPtrStore(base, element->value, (Pointer) NULL);
+	HnswPtrStore(base, element->itup, (IndexTuple) NULL);
 
 	return element;
 }
@@ -286,6 +353,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
 	element->offno = offno;
 	HnswPtrStore(base, element->neighbors, (HnswNeighborArrayPtr *) NULL);
 	HnswPtrStore(base, element->value, (Pointer) NULL);
+	HnswPtrStore(base, element->itup, (IndexTuple) NULL);
 	return element;
 }
 
@@ -401,10 +469,8 @@ HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, Bloc
  * Set element tuple, except for neighbor info
  */
 void
-HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element)
+HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element, bool useIndexTuple)
 {
-	Pointer		valuePtr = HnswPtrAccess(base, element->value);
-
 	etup->type = HNSW_ELEMENT_TUPLE_TYPE;
 	etup->level = element->level;
 	etup->deleted = 0;
@@ -415,7 +481,19 @@ HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element)
 		else
 			ItemPointerSetInvalid(&etup->heaptids[i]);
 	}
-	memcpy(&etup->data, valuePtr, VARSIZE_ANY(valuePtr));
+
+	if (useIndexTuple)
+	{
+		IndexTuple	itup = HnswPtrAccess(base, element->itup);
+
+		memcpy(&etup->data, itup, IndexTupleSize(itup));
+	}
+	else
+	{
+		Pointer		valuePtr = HnswPtrAccess(base, element->value);
+
+		memcpy(&etup->data, valuePtr, VARSIZE_ANY(valuePtr));
+	}
 }
 
 /*
@@ -452,74 +530,11 @@ HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m)
 	ntup->count = idx;
 }
 
-/*
- * Load neighbors from page
- */
-static void
-LoadNeighborsFromPage(HnswElement element, Relation index, Page page, int m)
-{
-	char	   *base = NULL;
-
-	HnswNeighborTuple ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
-	int			neighborCount = (element->level + 2) * m;
-
-	Assert(HnswIsNeighborTuple(ntup));
-
-	HnswInitNeighbors(base, element, m, NULL);
-
-	/* Ensure expected neighbors */
-	if (ntup->count != neighborCount)
-		return;
-
-	for (int i = 0; i < neighborCount; i++)
-	{
-		HnswElement e;
-		int			level;
-		HnswCandidate *hc;
-		ItemPointer indextid;
-		HnswNeighborArray *neighbors;
-
-		indextid = &ntup->indextids[i];
-
-		if (!ItemPointerIsValid(indextid))
-			continue;
-
-		e = HnswInitElementFromBlock(ItemPointerGetBlockNumber(indextid), ItemPointerGetOffsetNumber(indextid));
-
-		/* Calculate level based on offset */
-		level = element->level - i / m;
-		if (level < 0)
-			level = 0;
-
-		neighbors = HnswGetNeighbors(base, element, level);
-		hc = &neighbors->items[neighbors->length++];
-		HnswPtrStore(base, hc->element, e);
-	}
-}
-
-/*
- * Load neighbors
- */
-void
-HnswLoadNeighbors(HnswElement element, Relation index, int m)
-{
-	Buffer		buf;
-	Page		page;
-
-	buf = ReadBuffer(index, element->neighborPage);
-	LockBuffer(buf, BUFFER_LOCK_SHARE);
-	page = BufferGetPage(buf);
-
-	LoadNeighborsFromPage(element, index, page, m);
-
-	UnlockReleaseBuffer(buf);
-}
-
 /*
  * Load an element from a tuple
  */
 void
-HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec)
+HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec, Relation index)
 {
 	element->level = etup->level;
 	element->deleted = etup->deleted;
@@ -542,17 +557,128 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
 	if (loadVec)
 	{
 		char	   *base = NULL;
-		Datum		value = datumCopy(PointerGetDatum(&etup->data), false, -1);
 
-		HnswPtrStore(base, element->value, DatumGetPointer(value));
+		if (HnswUseIndexTuple(index))
+		{
+			IndexTuple	itup = CopyIndexTuple((IndexTuple) &etup->data);
+			TupleDesc	tupdesc = RelationGetDescr(index);
+			bool		unused;
+
+			HnswPtrStore(base, element->itup, itup);
+			HnswPtrStore(base, element->value, DatumGetPointer(index_getattr(itup, 1, tupdesc, &unused)));
+		}
+		else
+		{
+			Datum		value = datumCopy(PointerGetDatum(&etup->data), false, -1);
+
+			HnswPtrStore(base, element->value, DatumGetPointer(value));
+		}
 	}
 }
 
+/*
+ * Get the attribute distance
+ */
+static inline double
+AttributeDistance(double e)
+{
+	/* TODO Better bias */
+	/* must be >> max(w * g) + 1 / log10(2) */
+	double		bias = 4.32;
+
+	return e > 0 ? bias - 1.0 / log10(e + 1) : 0;
+}
+
+/*
+ * Calculate the distance between values
+ */
+static double
+HnswGetDistance(IndexTuple itup, Datum vec, Datum q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation, bool *matches)
+{
+	double		g;
+
+	if (DatumGetPointer(q) == NULL)
+		g = 0;
+	else
+		g = DatumGetFloat8(FunctionCall2Coll(procinfo[0], collation[0], q, vec));
+
+	Assert(PointerIsValid(matches));
+	*matches = true;
+
+	if (IndexRelationGetNumberOfKeyAttributes(index) > 1)
+	{
+		double		w = 0.25;
+		double		e = 0.0;
+		TupleDesc	tupdesc = RelationGetDescr(index);
+
+		if (keyData)
+		{
+			/* TODO need to pass length of key data */
+			int			keyCount = 1;
+
+			for (int i = 0; i < keyCount; i++)
+			{
+				ScanKey		key = &keyData[i];
+				bool		isnull;
+				Datum		value = index_getattr(itup, key->sk_attno, tupdesc, &isnull);
+				bool		attnull = key->sk_flags & SK_ISNULL;
+
+				if (isnull || attnull)
+				{
+					if (isnull != attnull)
+					{
+						e += 1000;
+						*matches = false;
+					}
+				}
+				else if (!DatumGetBool(FunctionCall2Coll(&key->sk_func, key->sk_collation, value, key->sk_argument)))
+				{
+					double		ei = fabs(DatumGetFloat8(FunctionCall2Coll(procinfo[key->sk_attno - 1], collation[key->sk_attno - 1], value, key->sk_argument)));
+
+					if (ei > 0)
+						e += ei;
+					else
+						/* Distance is zero for inequality */
+						e += 1000;
+
+					*matches = false;
+				}
+			}
+
+			return w * g + AttributeDistance(e);
+		}
+		else if (qtup)
+		{
+			int			keyCount = IndexRelationGetNumberOfKeyAttributes(index) - 1;
+
+			for (int i = 0; i < keyCount; i++)
+			{
+				bool		isnull;
+				bool		attnull;
+				Datum		value = index_getattr(itup, i + 2, tupdesc, &isnull);
+				Datum		value2 = index_getattr(qtup, i + 2, tupdesc, &attnull);
+
+				if (isnull || attnull)
+				{
+					if (isnull != attnull)
+						e += 1000;
+				}
+				else
+					e += fabs(DatumGetFloat8(FunctionCall2Coll(procinfo[i + 1], collation[i + 1], value, value2)));
+			}
+
+			return w * g + AttributeDistance(e);
+		}
+	}
+
+	return g;
+}
+
 /*
  * Load an element and optionally get its distance from q
  */
 static void
-HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance, HnswElement * element)
+HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, bool *matches, Datum *q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation, bool loadVec, double *maxDistance, HnswElement * element)
 {
 	Buffer		buf;
 	Page		page;
@@ -570,10 +696,23 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datu
 	/* Calculate distance */
 	if (distance != NULL)
 	{
-		if (DatumGetPointer(*q) == NULL)
-			*distance = 0;
+		IndexTuple	itup = NULL;
+		Datum		value;
+
+		if (HnswUseIndexTuple(index))
+		{
+			TupleDesc	tupdesc = RelationGetDescr(index);
+			bool		unused;
+
+			itup = (IndexTuple) &etup->data;
+			value = index_getattr(itup, 1, tupdesc, &unused);
+		}
 		else
-			*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
+		{
+			value = PointerGetDatum(&etup->data);
+		}
+
+		*distance = HnswGetDistance(itup, value, *q, qtup, keyData, index, procinfo, collation, matches);
 	}
 
 	/* Load element */
@@ -582,7 +721,7 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datu
 		if (*element == NULL)
 			*element = HnswInitElementFromBlock(blkno, offno);
 
-		HnswLoadElementFromTuple(*element, etup, true, loadVec);
+		HnswLoadElementFromTuple(*element, etup, true, loadVec, index);
 	}
 
 	UnlockReleaseBuffer(buf);
@@ -592,36 +731,37 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datu
  * Load an element and optionally get its distance from q
  */
 void
-HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance)
+HnswLoadElement(HnswElement element, double *distance, bool *matches, Datum *q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation, bool loadVec, double *maxDistance)
 {
-	HnswLoadElementImpl(element->blkno, element->offno, distance, q, index, procinfo, collation, loadVec, maxDistance, &element);
+	HnswLoadElementImpl(element->blkno, element->offno, distance, matches, q, qtup, keyData, index, procinfo, collation, loadVec, maxDistance, &element);
 }
 
 /*
  * Get the distance for an element
  */
-static float
-GetElementDistance(char *base, HnswElement element, Datum q, FmgrInfo *procinfo, Oid collation)
+static double
+GetElementDistance(char *base, HnswElement element, bool *matches, Datum q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation)
 {
 	Datum		value = HnswGetValue(base, element);
+	IndexTuple	itup = HnswPtrAccess(base, element->itup);
 
-	return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, value));
+	return HnswGetDistance(itup, value, q, qtup, keyData, index, procinfo, collation, matches);
 }
 
 /*
  * Create a candidate for the entry point
  */
 HnswSearchCandidate *
-HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
+HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation, bool loadVec, bool inMemory)
 {
-	HnswSearchCandidate *hc = palloc(sizeof(HnswSearchCandidate));
+	HnswSearchCandidate *sc = palloc(sizeof(HnswSearchCandidate));
 
-	HnswPtrStore(base, hc->element, entryPoint);
-	if (index == NULL)
-		hc->distance = GetElementDistance(base, entryPoint, q, procinfo, collation);
+	HnswPtrStore(base, sc->element, entryPoint);
+	if (inMemory)
+		sc->distance = GetElementDistance(base, entryPoint, &sc->matches, q, qtup, keyData, index, procinfo, collation);
 	else
-		HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec, NULL);
-	return hc;
+		HnswLoadElement(entryPoint, &sc->distance, &sc->matches, &q, qtup, keyData, index, procinfo, collation, loadVec, NULL);
+	return sc;
 }
 
 #define HnswGetSearchCandidate(membername, ptr) pairingheap_container(HnswSearchCandidate, membername, ptr)
@@ -661,9 +801,9 @@ CompareFurthestCandidates(const pairingheap_node *a, const pairingheap_node *b,
  * Init visited
  */
 static inline void
-InitVisited(char *base, visited_hash * v, Relation index, int ef, int m)
+InitVisited(char *base, visited_hash * v, bool inMemory, int ef, int m)
 {
-	if (index != NULL)
+	if (!inMemory)
 		v->tids = tidhash_create(CurrentMemoryContext, ef * m * 2, NULL);
 	else if (base != NULL)
 		v->offsets = offsethash_create(CurrentMemoryContext, ef * m * 2, NULL);
@@ -675,9 +815,9 @@ InitVisited(char *base, visited_hash * v, Relation index, int ef, int m)
  * Add to visited
  */
 static inline void
-AddToVisited(char *base, visited_hash * v, HnswElementPtr elementPtr, Relation index, bool *found)
+AddToVisited(char *base, visited_hash * v, HnswElementPtr elementPtr, bool inMemory, bool *found)
 {
-	if (index != NULL)
+	if (!inMemory)
 	{
 		HnswElement element = HnswPtrAccess(base, elementPtr);
 		ItemPointerData indextid;
@@ -738,39 +878,58 @@ HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unv
 		HnswCandidate *hc = &localNeighborhood->items[i];
 		bool		found;
 
-		AddToVisited(base, v, hc->element, NULL, &found);
+		AddToVisited(base, v, hc->element, true, &found);
 
 		if (!found)
 			unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
 	}
 }
 
+/*
+ * Load neighbor index TIDs
+ */
+bool
+HnswLoadNeighborTids(HnswElement element, ItemPointerData *indextids, Relation index, int m, int lm, int lc)
+{
+	Buffer		buf;
+	Page		page;
+	HnswNeighborTuple ntup;
+	int			start;
+
+	buf = ReadBuffer(index, element->neighborPage);
+	LockBuffer(buf, BUFFER_LOCK_SHARE);
+	page = BufferGetPage(buf);
+
+	ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
+
+	/* Ensure expected neighbors */
+	if (ntup->count != (element->level + 2) * m)
+	{
+		UnlockReleaseBuffer(buf);
+		return false;
+	}
+
+	/* Copy to minimize lock time */
+	start = (element->level - lc) * m;
+	memcpy(indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
+
+	UnlockReleaseBuffer(buf);
+	return true;
+}
+
 /*
  * Load unvisited neighbors from disk
  */
 static void
 HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, Relation index, int m, int lm, int lc)
 {
-	Buffer		buf;
-	Page		page;
-	HnswNeighborTuple ntup;
-	int			start;
 	ItemPointerData indextids[HNSW_MAX_M * 2];
 
-	buf = ReadBuffer(index, element->neighborPage);
-	LockBuffer(buf, BUFFER_LOCK_SHARE);
-	page = BufferGetPage(buf);
-
-	ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
-	start = (element->level - lc) * m;
-
-	/* Copy to minimize lock time */
-	memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
-
-	UnlockReleaseBuffer(buf);
-
 	*unvisitedLength = 0;
 
+	if (!HnswLoadNeighborTids(element, indextids, index, m, lm, lc))
+		return;
+
 	for (int i = 0; i < lm; i++)
 	{
 		ItemPointer indextid = &indextids[i];
@@ -790,7 +949,7 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
  * Algorithm 2 from paper
  */
 List *
-HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
+HnswSearchLayer(char *base, Datum q, IndexTuple qtup, ScanKeyData *keyData, List *ep, int ef, int lc, Relation index, FmgrInfo **procinfo, Oid *collation, int m, bool inserting, HnswElement skipElement, bool inMemory)
 {
 	List	   *w = NIL;
 	pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
@@ -803,11 +962,13 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 	int			lm = HnswGetLayerM(m, lc);
 	HnswUnvisited *unvisited = palloc(lm * sizeof(HnswUnvisited));
 	int			unvisitedLength;
+	uint64		additional = 0;
+	uint64		maxAdditional = keyData && lc == 0 ? 10000 : 0;
 
-	InitVisited(base, &v, index, ef, m);
+	InitVisited(base, &v, inMemory, ef, m);
 
 	/* Create local memory for neighborhood if needed */
-	if (index == NULL)
+	if (inMemory)
 	{
 		neighborhoodSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm);
 		localNeighborhood = palloc(neighborhoodSize);
@@ -816,20 +977,24 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 	/* Add entry points to v, C, and W */
 	foreach(lc2, ep)
 	{
-		HnswSearchCandidate *hc = (HnswSearchCandidate *) lfirst(lc2);
+		HnswSearchCandidate *sc = (HnswSearchCandidate *) lfirst(lc2);
 		bool		found;
 
-		AddToVisited(base, &v, hc->element, index, &found);
+		AddToVisited(base, &v, sc->element, inMemory, &found);
 
-		pairingheap_add(C, &hc->c_node);
-		pairingheap_add(W, &hc->w_node);
+		pairingheap_add(C, &sc->c_node);
+		pairingheap_add(W, &sc->w_node);
+
+		/* Do not count elements that do not match filter towards ef */
+		if (!sc->matches && ++additional <= maxAdditional)
+			continue;
 
 		/*
 		 * Do not count elements being deleted towards ef when vacuuming. It
 		 * would be ideal to do this for inserts as well, but this could
 		 * affect insert performance.
 		 */
-		if (CountElement(skipElement, HnswPtrAccess(base, hc->element)))
+		if (CountElement(skipElement, HnswPtrAccess(base, sc->element)))
 			wlen++;
 	}
 
@@ -844,7 +1009,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 
 		cElement = HnswPtrAccess(base, c->element);
 
-		if (index == NULL)
+		if (inMemory)
 			HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, &v, lc, localNeighborhood, neighborhoodSize);
 		else
 			HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, &v, index, m, lm, lc);
@@ -853,15 +1018,16 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 		{
 			HnswElement eElement;
 			HnswSearchCandidate *e;
-			float		eDistance;
+			double		eDistance;
+			bool		eMatches;
 			bool		alwaysAdd = wlen < ef;
 
 			f = HnswGetSearchCandidate(w_node, pairingheap_first(W));
 
-			if (index == NULL)
+			if (inMemory)
 			{
 				eElement = unvisited[i].element;
-				eDistance = GetElementDistance(base, eElement, q, procinfo, collation);
+				eDistance = GetElementDistance(base, eElement, &eMatches, q, qtup, keyData, index, procinfo, collation);
 			}
 			else
 			{
@@ -871,7 +1037,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 
 				/* Avoid any allocations if not adding */
 				eElement = NULL;
-				HnswLoadElementImpl(blkno, offno, &eDistance, &q, index, procinfo, collation, inserting, alwaysAdd ? NULL : &f->distance, &eElement);
+				HnswLoadElementImpl(blkno, offno, &eDistance, &eMatches, &q, qtup, keyData, index, procinfo, collation, inserting, alwaysAdd ? NULL : &f->distance, &eElement);
 
 				if (eElement == NULL)
 					continue;
@@ -890,6 +1056,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 			e = palloc(sizeof(HnswSearchCandidate));
 			HnswPtrStore(base, e->element, eElement);
 			e->distance = eDistance;
+			e->matches = eMatches;
 			pairingheap_add(C, &e->c_node);
 			pairingheap_add(W, &e->w_node);
 
@@ -900,6 +1067,10 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 			 */
 			if (CountElement(skipElement, eElement))
 			{
+				/* Do not count elements that do not match filter towards ef */
+				if (!e->matches && ++additional <= maxAdditional)
+					continue;
+
 				wlen++;
 
 				/* No need to decrement wlen */
@@ -912,9 +1083,9 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 	/* Add each element of W to w */
 	while (!pairingheap_is_empty(W))
 	{
-		HnswSearchCandidate *hc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
+		HnswSearchCandidate *sc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
 
-		w = lappend(w, hc);
+		w = lappend(w, sc);
 	}
 
 	return w;
@@ -968,32 +1139,25 @@ CompareCandidateDistancesOffset(const ListCell *a, const ListCell *b)
 	return 0;
 }
 
-/*
- * Calculate the distance between elements
- */
-static float
-HnswGetDistance(char *base, HnswElement a, HnswElement b, FmgrInfo *procinfo, Oid collation)
-{
-	Datum		aValue = HnswGetValue(base, a);
-	Datum		bValue = HnswGetValue(base, b);
-
-	return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, aValue, bValue));
-}
-
 /*
  * Check if an element is closer to q than any element from R
  */
 static bool
-CheckElementCloser(char *base, HnswCandidate * e, List *r, FmgrInfo *procinfo, Oid collation)
+CheckElementCloser(char *base, HnswCandidate * e, List *r, Relation index, FmgrInfo **procinfo, Oid *collation)
 {
 	HnswElement eElement = HnswPtrAccess(base, e->element);
+	Datum		eValue = HnswGetValue(base, eElement);
+	IndexTuple	etup = HnswPtrAccess(base, eElement->itup);
 	ListCell   *lc2;
 
 	foreach(lc2, r)
 	{
 		HnswCandidate *ri = lfirst(lc2);
 		HnswElement riElement = HnswPtrAccess(base, ri->element);
-		float		distance = HnswGetDistance(base, eElement, riElement, procinfo, collation);
+		Datum		riValue = HnswGetValue(base, riElement);
+		IndexTuple	ritup = HnswPtrAccess(base, riElement->itup);
+		bool		matches;
+		float		distance = HnswGetDistance(etup, eValue, riValue, ritup, NULL, index, procinfo, collation, &matches);
 
 		if (distance <= e->distance)
 			return false;
@@ -1006,15 +1170,14 @@ CheckElementCloser(char *base, HnswCandidate * e, List *r, FmgrInfo *procinfo, O
  * Algorithm 4 from paper
  */
 static List *
-SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid collation, HnswElement e2, HnswCandidate * newCandidate, HnswCandidate * *pruned, bool sortCandidates)
+SelectNeighbors(char *base, List *c, int lm, Relation index, FmgrInfo **procinfo, Oid *collation, bool *closerSet, HnswCandidate * newCandidate, HnswCandidate * *pruned, bool sortCandidates)
 {
 	List	   *r = NIL;
 	List	   *w = list_copy(c);
 	HnswCandidate **wd;
 	int			wdlen = 0;
 	int			wdoff = 0;
-	HnswNeighborArray *neighbors = HnswGetNeighbors(base, e2, lc);
-	bool		mustCalculate = !neighbors->closerSet;
+	bool		mustCalculate = !(*closerSet);
 	List	   *added = NIL;
 	bool		removedAny = false;
 
@@ -1041,7 +1204,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 
 		/* Use previous state of r and wd to skip work when possible */
 		if (mustCalculate)
-			e->closer = CheckElementCloser(base, e, r, procinfo, collation);
+			e->closer = CheckElementCloser(base, e, r, index, procinfo, collation);
 		else if (list_length(added) > 0)
 		{
 			/* Keep Valgrind happy for in-memory, parallel builds */
@@ -1054,7 +1217,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 			 */
 			if (e->closer)
 			{
-				e->closer = CheckElementCloser(base, e, added, procinfo, collation);
+				e->closer = CheckElementCloser(base, e, added, index, procinfo, collation);
 
 				if (!e->closer)
 					removedAny = true;
@@ -1067,7 +1230,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 				 */
 				if (removedAny)
 				{
-					e->closer = CheckElementCloser(base, e, r, procinfo, collation);
+					e->closer = CheckElementCloser(base, e, r, index, procinfo, collation);
 					if (e->closer)
 						added = lappend(added, e);
 				}
@@ -1075,7 +1238,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 		}
 		else if (e == newCandidate)
 		{
-			e->closer = CheckElementCloser(base, e, r, procinfo, collation);
+			e->closer = CheckElementCloser(base, e, r, index, procinfo, collation);
 			if (e->closer)
 				added = lappend(added, e);
 		}
@@ -1091,7 +1254,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
 	}
 
 	/* Cached value can only be used in future if sorted deterministically */
-	neighbors->closerSet = sortCandidates;
+	*closerSet = sortCandidates;
 
 	/* Keep pruned connections */
 	while (wdoff < wdlen && list_length(r) < lm)
@@ -1126,18 +1289,16 @@ AddConnections(char *base, HnswElement element, List *neighbors, int lc)
  * Update connections
  */
 void
-HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation)
+HnswUpdateConnection(char *base, HnswNeighborArray * neighbors, HnswElement newElement, float distance, int lm, int *updateIdx, Relation index, FmgrInfo **procinfo, Oid *collation)
 {
-	HnswElement hce = HnswPtrAccess(base, hc->element);
-	HnswNeighborArray *currentNeighbors = HnswGetNeighbors(base, hce, lc);
-	HnswCandidate hc2;
+	HnswCandidate newHc;
 
-	HnswPtrStore(base, hc2.element, element);
-	hc2.distance = hc->distance;
+	HnswPtrStore(base, newHc.element, newElement);
+	newHc.distance = distance;
 
-	if (currentNeighbors->length < lm)
+	if (neighbors->length < lm)
 	{
-		currentNeighbors->items[currentNeighbors->length++] = hc2;
+		neighbors->items[neighbors->length++] = newHc;
 
 		/* Track update */
 		if (updateIdx != NULL)
@@ -1146,54 +1307,26 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
 	else
 	{
 		/* Shrink connections */
+		List	   *c = NIL;
 		HnswCandidate *pruned = NULL;
 
-		/* Load elements on insert */
-		if (index != NULL)
-		{
-			Datum		q = HnswGetValue(base, hce);
+		/* Add candidates */
+		for (int i = 0; i < neighbors->length; i++)
+			c = lappend(c, &neighbors->items[i]);
+		c = lappend(c, &newHc);
 
-			for (int i = 0; i < currentNeighbors->length; i++)
-			{
-				HnswCandidate *hc3 = &currentNeighbors->items[i];
-				HnswElement hc3Element = HnswPtrAccess(base, hc3->element);
-
-				if (HnswPtrIsNull(base, hc3Element->value))
-					HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true, NULL);
-				else
-					hc3->distance = GetElementDistance(base, hc3Element, q, procinfo, collation);
-
-				/* Prune element if being deleted */
-				if (hc3Element->heaptidsLength == 0)
-				{
-					pruned = &currentNeighbors->items[i];
-					break;
-				}
-			}
-		}
+		SelectNeighbors(base, c, lm, index, procinfo, collation, &neighbors->closerSet, &newHc, &pruned, true);
 
+		/* Should not happen */
 		if (pruned == NULL)
-		{
-			List	   *c = NIL;
-
-			/* Add candidates */
-			for (int i = 0; i < currentNeighbors->length; i++)
-				c = lappend(c, &currentNeighbors->items[i]);
-			c = lappend(c, &hc2);
-
-			SelectNeighbors(base, c, lm, lc, procinfo, collation, hce, &hc2, &pruned, true);
-
-			/* Should not happen */
-			if (pruned == NULL)
-				return;
-		}
+			return;
 
 		/* Find and replace the pruned element */
-		for (int i = 0; i < currentNeighbors->length; i++)
+		for (int i = 0; i < neighbors->length; i++)
 		{
-			if (HnswPtrEqual(base, currentNeighbors->items[i].element, pruned->element))
+			if (HnswPtrEqual(base, neighbors->items[i].element, pruned->element))
 			{
-				currentNeighbors->items[i] = hc2;
+				neighbors->items[i] = newHc;
 
 				/* Track update */
 				if (updateIdx != NULL)
@@ -1253,17 +1386,19 @@ PrecomputeHash(char *base, HnswElement element)
  * Algorithm 1 from paper
  */
 void
-HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing)
+HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo **procinfo, Oid *collation, int m, int efConstruction, bool existing, bool inMemory)
 {
 	List	   *ep;
 	List	   *w;
 	int			level = element->level;
 	int			entryLevel;
 	Datum		q = HnswGetValue(base, element);
+	IndexTuple	qtup = HnswPtrAccess(base, element->itup);
+	ScanKeyData *keyData = NULL;
 	HnswElement skipElement = existing ? element : NULL;
 
 	/* Precompute hash */
-	if (index == NULL)
+	if (inMemory)
 		PrecomputeHash(base, element);
 
 	/* No neighbors if no entry point */
@@ -1271,13 +1406,13 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		return;
 
 	/* Get entry point and level */
-	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, true));
+	ep = list_make1(HnswEntryCandidate(base, entryPoint, q, qtup, keyData, index, procinfo, collation, true, inMemory));
 	entryLevel = entryPoint->level;
 
 	/* 1st phase: greedy search to insert level */
 	for (int lc = entryLevel; lc >= level + 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
+		w = HnswSearchLayer(base, q, qtup, keyData, ep, 1, lc, index, procinfo, collation, m, true, skipElement, inMemory);
 		ep = w;
 	}
 
@@ -1296,7 +1431,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		List	   *lw = NIL;
 		ListCell   *lc2;
 
-		w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
+		w = HnswSearchLayer(base, q, qtup, keyData, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement, inMemory);
 
 		/* Convert search candidates to candidates */
 		foreach(lc2, w)
@@ -1312,7 +1447,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 
 		/* Elements being deleted or skipped can help with search */
 		/* but should be removed before selecting neighbors */
-		if (index != NULL)
+		if (!inMemory)
 			lw = RemoveElements(base, lw, skipElement);
 
 		/*
@@ -1320,7 +1455,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		 * sortCandidates to true for in-memory builds to enable closer
 		 * caching, but there does not seem to be a difference in performance.
 		 */
-		neighbors = SelectNeighbors(base, lw, lm, lc, procinfo, collation, element, NULL, NULL, false);
+		neighbors = SelectNeighbors(base, lw, lm, index, procinfo, collation, &HnswGetNeighbors(base, element, lc)->closerSet, NULL, NULL, false);
 
 		AddConnections(base, element, neighbors, lc);
 
@@ -1330,7 +1465,6 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 
 PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
 PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
-PGDLLEXPORT Datum minivec_l2_normalize(PG_FUNCTION_ARGS);
 PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
 
 static void
@@ -1379,20 +1513,6 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(&typeInfo);
 };
 
-FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_minivec_support);
-Datum
-hnsw_minivec_support(PG_FUNCTION_ARGS)
-{
-	static const HnswTypeInfo typeInfo = {
-		.maxDimensions = HNSW_MAX_DIM * 4,
-		/* Do not normalize to maximize precision */
-		.normalize = NULL,
-		.checkValue = NULL
-	};
-
-	PG_RETURN_POINTER(&typeInfo);
-};
-
 FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
 Datum
 hnsw_bit_support(PG_FUNCTION_ARGS)

src/hnswvacuum.c

@@ -189,8 +189,8 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
 	GenericXLogState *state;
 	int			m = vacuumstate->m;
 	int			efConstruction = vacuumstate->efConstruction;
-	FmgrInfo   *procinfo = vacuumstate->procinfo;
-	Oid			collation = vacuumstate->collation;
+	FmgrInfo  **procinfo = vacuumstate->procinfo;
+	Oid		   *collation = vacuumstate->collation;
 	BufferAccessStrategy bas = vacuumstate->bas;
 	HnswNeighborTuple ntup = vacuumstate->ntup;
 	Size		ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
@@ -205,7 +205,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
 	element->heaptidsLength = 0;
 
 	/* Find neighbors for element, skipping itself */
-	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
+	HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true, false);
 
 	/* Zero memory for each element */
 	MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
@@ -256,7 +256,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
 		LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
 
 		/* Load element */
-		HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL);
+		HnswLoadElement(highestPoint, NULL, NULL, NULL, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL);
 
 		/* Repair if needed */
 		if (NeedsUpdated(vacuumstate, highestPoint))
@@ -294,7 +294,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
 			 * is outdated, this can remove connections at higher levels in
 			 * the graph until they are repaired, but this should be fine.
 			 */
-			HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL);
+			HnswLoadElement(entryPoint, NULL, NULL, NULL, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL);
 
 			if (NeedsUpdated(vacuumstate, entryPoint))
 			{
@@ -370,7 +370,7 @@ RepairGraph(HnswVacuumState * vacuumstate)
 
 			/* Create an element */
 			element = HnswInitElementFromBlock(blkno, offno);
-			HnswLoadElementFromTuple(element, etup, false, true);
+			HnswLoadElementFromTuple(element, etup, false, true, index);
 
 			elements = lappend(elements, element);
 		}
@@ -440,6 +440,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 	BlockNumber insertPage = InvalidBlockNumber;
 	Relation	index = vacuumstate->index;
 	BufferAccessStrategy bas = vacuumstate->bas;
+	bool		useIndexTuple = HnswUseIndexTuple(index);
 
 	/*
 	 * Wait for index scans to complete. Scans before this point may contain
@@ -521,7 +522,14 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 
 			/* Overwrite element */
 			etup->deleted = 1;
-			MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
+			if (useIndexTuple)
+			{
+				IndexTuple	itup = (IndexTuple) &etup->data;
+
+				MemSet(itup, 0, IndexTupleSize(itup));
+			}
+			else
+				MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
 
 			/* Overwrite neighbors */
 			for (int i = 0; i < ntup->count; i++)
@@ -573,8 +581,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 	vacuumstate->callback_state = callback_state;
 	vacuumstate->efConstruction = HnswGetEfConstruction(index);
 	vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
-	vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
-	vacuumstate->collation = index->rd_indcollation[0];
+	HnswInitProcinfo(vacuumstate->procinfo, &vacuumstate->collation, index);
 	vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
 	vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
 												"Hnsw vacuum temporary context",

src/ivfflat.c

@@ -69,6 +69,8 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	GenericCosts costs;
 	int			lists;
 	double		ratio;
+	double		sequentialRatio = 0.5;
+	double		startupPages;
 	double		spc_seq_page_cost;
 	Relation	index;
 
@@ -85,6 +87,8 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 
 	MemSet(&costs, 0, sizeof(costs));
 
+	genericcostestimate(root, path, loop_count, &costs);
+
 	index = index_open(path->indexinfo->indexoid, NoLock);
 	IvfflatGetMetaPageInfo(index, &lists, NULL);
 	index_close(index, NoLock);
@@ -94,41 +98,26 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	if (ratio > 1.0)
 		ratio = 1.0;
 
-	/*
-	 * This gives us the subset of tuples to visit. This value is passed into
-	 * the generic cost estimator to determine the number of pages to visit
-	 * during the index scan.
-	 */
-	costs.numIndexTuples = path->indexinfo->tuples * ratio;
-
-	genericcostestimate(root, path, loop_count, &costs);
-
 	get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
 
+	/* Change some page cost from random to sequential */
+	costs.indexTotalCost -= sequentialRatio * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+
+	/* Startup cost is cost before returning the first row */
+	costs.indexStartupCost = costs.indexTotalCost * ratio;
+
 	/* Adjust cost if needed since TOAST not included in seq scan cost */
-	if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
+	startupPages = costs.numIndexPages * ratio;
+	if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
 	{
-		/* Change all page cost from random to sequential */
-		costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+		/* Change rest of page cost from random to sequential */
+		costs.indexStartupCost -= (1 - sequentialRatio) * startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
 
 		/* Remove cost of extra pages */
-		costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
-	}
-	else
-	{
-		/* Change some page cost from random to sequential */
-		costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+		costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
 	}
 
-	/*
-	 * If the list selectivity is lower than what is returned from the generic
-	 * cost estimator, use that.
-	 */
-	if (ratio < costs.indexSelectivity)
-		costs.indexSelectivity = ratio;
-
-	/* Use total cost since most work happens before first tuple is returned */
-	*indexStartupCost = costs.indexTotalCost;
+	*indexStartupCost = costs.indexStartupCost;
 	*indexTotalCost = costs.indexTotalCost;
 	*indexSelectivity = costs.indexSelectivity;
 	*indexCorrelation = costs.indexCorrelation;

src/ivfutils.c

@@ -7,7 +7,6 @@
 #include "halfutils.h"
 #include "halfvec.h"
 #include "ivfflat.h"
-#include "minivec.h"
 #include "storage/bufmgr.h"
 
 /*
@@ -71,9 +70,6 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
 Datum
 IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value)
 {
-	if (!typeInfo->normalize)
-		return value;
-
 	return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
 }
 
@@ -235,7 +231,6 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
 
 PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
 PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
-PGDLLEXPORT Datum minivec_l2_normalize(PG_FUNCTION_ARGS);
 PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
 
 static Size
@@ -250,12 +245,6 @@ HalfvecItemSize(int dimensions)
 	return HALFVEC_SIZE(dimensions);
 }
 
-static Size
-MinivecItemSize(int dimensions)
-{
-	return MINIVEC_SIZE(dimensions);
-}
-
 static Size
 BitItemSize(int dimensions)
 {
@@ -286,18 +275,6 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
 		vec->x[k] = Float4ToHalfUnchecked(x[k]);
 }
 
-static void
-MinivecUpdateCenter(Pointer v, int dimensions, float *x)
-{
-	MiniVector *vec = (MiniVector *) v;
-
-	SET_VARSIZE(vec, MINIVEC_SIZE(dimensions));
-	vec->dim = dimensions;
-
-	for (int k = 0; k < dimensions; k++)
-		vec->x[k] = Float4ToFp8Unchecked(x[k]);
-}
-
 static void
 BitUpdateCenter(Pointer v, int dimensions, float *x)
 {
@@ -332,15 +309,6 @@ HalfvecSumCenter(Pointer v, float *x)
 		x[k] += HalfToFloat4(vec->x[k]);
 }
 
-static void
-MinivecSumCenter(Pointer v, float *x)
-{
-	MiniVector *vec = (MiniVector *) v;
-
-	for (int k = 0; k < vec->dim; k++)
-		x[k] += Fp8ToFloat4(vec->x[k]);
-}
-
 static void
 BitSumCenter(Pointer v, float *x)
 {
@@ -389,22 +357,6 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(&typeInfo);
 };
 
-FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_minivec_support);
-Datum
-ivfflat_minivec_support(PG_FUNCTION_ARGS)
-{
-	static const IvfflatTypeInfo typeInfo = {
-		.maxDimensions = IVFFLAT_MAX_DIM * 4,
-		/* Do not normalize to maximize precision */
-		.normalize = NULL,
-		.itemSize = MinivecItemSize,
-		.updateCenter = MinivecUpdateCenter,
-		.sumCenter = MinivecSumCenter
-	};
-
-	PG_RETURN_POINTER(&typeInfo);
-};
-
 FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
 Datum
 ivfflat_bit_support(PG_FUNCTION_ARGS)

src/minivec.h

@@ -1,177 +0,0 @@
-#ifndef MINIVEC_H
-#define MINIVEC_H
-
-#define MINIVEC_MAX_DIM 16000
-
-#define fp8 uint8
-
-#define MINIVEC_SIZE(_dim)		(offsetof(MiniVector, x) + sizeof(fp8)*(_dim))
-#define DatumGetMiniVector(x)		((MiniVector *) PG_DETOAST_DATUM(x))
-#define PG_GETARG_MINIVEC_P(x)	DatumGetMiniVector(PG_GETARG_DATUM(x))
-#define PG_RETURN_MINIVEC_P(x)	PG_RETURN_POINTER(x)
-
-typedef struct MiniVector
-{
-	int32		vl_len_;		/* varlena header (do not touch directly!) */
-	int16		dim;			/* number of dimensions */
-	int16		unused;			/* reserved for future use, always zero */
-	fp8			x[FLEXIBLE_ARRAY_MEMBER];
-}			MiniVector;
-
-MiniVector *InitMiniVector(int dim);
-
-/*
- * Check if fp8 is NaN
- */
-static inline bool
-Fp8IsNan(fp8 num)
-{
-	return (num & 0x7C) == 0x7C && (num & 0x7F) != 0x7C;
-}
-
-/*
- * Check if fp8 is infinite
- */
-static inline bool
-Fp8IsInf(fp8 num)
-{
-	return (num & 0x7F) == 0x7C;
-}
-
-/*
- * Check if fp8 is zero
- */
-static inline bool
-Fp8IsZero(fp8 num)
-{
-	return num == 0;
-}
-
-/*
- * Convert a fp8 to a float4
- */
-static inline float
-Fp8ToFloat4(fp8 num)
-{
-	/* Lookup table for non-sign bits */
-	/* Uses uint32 for correctness */
-	uint32		lookup[128] = {0, 931135488, 939524096, 943718400, 947912704, 950009856, 952107008, 954204160, 956301312, 958398464, 960495616, 962592768, 964689920, 966787072, 968884224, 970981376, 973078528, 975175680, 977272832, 979369984, 981467136, 983564288, 985661440, 987758592, 989855744, 991952896, 994050048, 996147200, 998244352, 1000341504, 1002438656, 1004535808, 1006632960, 1008730112, 1010827264, 1012924416, 1015021568, 1017118720, 1019215872, 1021313024, 1023410176, 1025507328, 1027604480, 1029701632, 1031798784, 1033895936, 1035993088, 1038090240, 1040187392, 1042284544, 1044381696, 1046478848, 1048576000, 1050673152, 1052770304, 1054867456, 1056964608, 1059061760, 1061158912, 1063256064, 1065353216, 1067450368, 1069547520, 1071644672, 1073741824, 1075838976, 1077936128, 1080033280, 1082130432, 1084227584, 1086324736, 1088421888, 1090519040, 1092616192, 1094713344, 1096810496, 1098907648, 1101004800, 1103101952, 1105199104, 1107296256, 1109393408, 1111490560, 1113587712, 1115684864, 1117782016, 1119879168, 1121976320, 1124073472, 1126170624, 1128267776, 1130364928, 1132462080, 1134559232, 1136656384, 1138753536, 1140850688, 1142947840, 1145044992, 1147142144, 1149239296, 1151336448, 1153433600, 1155530752, 1157627904, 1159725056, 1161822208, 1163919360, 1166016512, 1168113664, 1170210816, 1172307968, 1174405120, 1176502272, 1178599424, 1180696576, 1182793728, 1184890880, 1186988032, 1189085184, 1191182336, 1193279488, 1195376640, 1197473792, 2139095040, 2145386496, 2143289344, 2145386496};
-
-	union
-	{
-		float		f;
-		uint32		i;
-	}			swap;
-
-	swap.i = lookup[num & 0x7F];
-
-	return (num & 0x80) == 0x80 ? -swap.f : swap.f;
-}
-
-/*
- * Convert a float4 to a fp8
- */
-static inline fp8
-Float4ToFp8Unchecked(float num)
-{
-	union
-	{
-		float		f;
-		uint32		i;
-	}			swap;
-
-	uint32		bin;
-	int			exponent;
-	int			mantissa;
-	uint8		result;
-
-	swap.f = num;
-	bin = swap.i;
-	exponent = (bin & 0x7F800000) >> 23;
-	mantissa = bin & 0x007FFFFF;
-
-	/* Sign */
-	result = (bin & 0x80000000) >> 24;
-
-	if (isinf(num))
-	{
-		/* Infinite */
-		result |= 0x7C;
-	}
-	else if (isnan(num))
-	{
-		/* NaN */
-		result |= 0x7F;
-	}
-	else if (exponent > 98)
-	{
-		int			m;
-		int			gr;
-		int			s;
-
-		exponent -= 127;
-		s = mantissa & 0x000FFFFF;
-
-		/* Subnormal */
-		if (exponent < -14)
-		{
-			int			diff = -exponent - 14;
-
-			mantissa >>= diff;
-			mantissa += 1 << (23 - diff);
-			s |= mantissa & 0x000FFFFF;
-		}
-
-		m = mantissa >> 21;
-
-		/* Round */
-		gr = (mantissa >> 20) % 4;
-		if (gr == 3 || (gr == 1 && s != 0))
-			m += 1;
-
-		if (m == 4)
-		{
-			m = 0;
-			exponent += 1;
-		}
-
-		if (exponent > 15)
-		{
-			/* Infinite */
-			result |= 0x7C;
-		}
-		else
-		{
-			if (exponent >= -14)
-				result |= (exponent + 15) << 2;
-
-			result |= m;
-		}
-	}
-
-	return result;
-}
-
-/*
- * Convert a float4 to a fp8
- */
-static inline fp8
-Float4ToFp8(float num)
-{
-	fp8			result = Float4ToFp8Unchecked(num);
-
-	if (unlikely(Fp8IsInf(result)) && !isinf(num))
-	{
-		char	   *buf = palloc(FLOAT_SHORTEST_DECIMAL_LEN);
-
-		float_to_shortest_decimal_buf(num, buf);
-
-		ereport(ERROR,
-				(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
-				 errmsg("\"%s\" is out of range for type minivec", buf)));
-	}
-
-	return result;
-}
-
-#endif

src/vector.c

@@ -13,7 +13,6 @@
 #include "ivfflat.h"
 #include "lib/stringinfo.h"
 #include "libpq/pqformat.h"
-#include "minivec.h"
 #include "port.h"				/* for strtof() */
 #include "sparsevec.h"
 #include "utils/array.h"
@@ -543,28 +542,6 @@ halfvec_to_vector(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(result);
 }
 
-/*
- * Convert fp8 vector to vector
- */
-FUNCTION_PREFIX PG_FUNCTION_INFO_V1(minivec_to_vector);
-Datum
-minivec_to_vector(PG_FUNCTION_ARGS)
-{
-	MiniVector *vec = PG_GETARG_MINIVEC_P(0);
-	int32		typmod = PG_GETARG_INT32(1);
-	Vector	   *result;
-
-	CheckDim(vec->dim);
-	CheckExpectedDim(typmod, vec->dim);
-
-	result = InitVector(vec->dim);
-
-	for (int i = 0; i < vec->dim; i++)
-		result->x[i] = Fp8ToFloat4(vec->x[i]);
-
-	PG_RETURN_POINTER(result);
-}
-
 VECTOR_TARGET_CLONES static float
 VectorL2SquaredDistance(int dim, float *ax, float *bx)
 {

test/expected/btree.out

@@ -38,26 +38,6 @@ SELECT * FROM t ORDER BY val;
  
 (4 rows)
 
-DROP TABLE t;
--- minivec
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t (val);
-SELECT * FROM t WHERE val = '[1,2,3]';
-   val   
----------
- [1,2,3]
-(1 row)
-
-SELECT * FROM t ORDER BY val;
-   val   
----------
- [0,0,0]
- [1,1,1]
- [1,2,3]
- 
-(4 rows)
-
 DROP TABLE t;
 -- sparsevec
 CREATE TABLE t (val sparsevec(3));

test/expected/cast.out

@@ -140,64 +140,6 @@ SELECT '{1e-8,-1e-8}'::real[]::halfvec;
  [0,-0]
 (1 row)
 
-SELECT '[1,2,3]'::vector::minivec;
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '[1,2,3]'::vector::minivec(3);
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '[1,2,3]'::vector::minivec(2);
-ERROR:  expected 2 dimensions, not 3
-SELECT '[61440]'::vector::minivec;
-ERROR:  "61440" is out of range for type minivec
-SELECT '[1e-8]'::vector::minivec;
- minivec 
----------
- [0]
-(1 row)
-
-SELECT '[1,2,3]'::minivec::vector;
- vector  
----------
- [1,2,3]
-(1 row)
-
-SELECT '[1,2,3]'::minivec::vector(3);
- vector  
----------
- [1,2,3]
-(1 row)
-
-SELECT '[1,2,3]'::minivec::vector(2);
-ERROR:  expected 2 dimensions, not 3
-SELECT '{1,2,3}'::real[]::minivec;
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '{1,2,3}'::real[]::minivec(3);
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '{1,2,3}'::real[]::minivec(2);
-ERROR:  expected 2 dimensions, not 3
-SELECT '{61440,-61440}'::real[]::minivec;
-ERROR:  "61440" is out of range for type minivec
-SELECT '{1e-8,-1e-8}'::real[]::minivec;
- minivec 
----------
- [0,-0]
-(1 row)
-
 SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
     sparsevec    
 -----------------

test/expected/copy.out

@@ -30,23 +30,6 @@ SELECT * FROM t2 ORDER BY val;
  
 (4 rows)
 
-DROP TABLE t;
-DROP TABLE t2;
--- minivec
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE TABLE t2 (val minivec(3));
-\copy t TO 'results/minivec.bin' WITH (FORMAT binary)
-\copy t2 FROM 'results/minivec.bin' WITH (FORMAT binary)
-SELECT * FROM t2 ORDER BY val;
-   val   
----------
- [0,0,0]
- [1,1,1]
- [1,2,3]
- 
-(4 rows)
-
 DROP TABLE t;
 DROP TABLE t2;
 -- sparsevec

test/expected/ivfflat_minivec.out

@@ -1,84 +0,0 @@
-SET enable_seqscan = off;
--- L2
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val minivec_l2_ops) WITH (lists = 1);
-INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-   val   
----------
- [1,2,3]
- [1,2,4]
- [1,1,1]
- [0,0,0]
-(4 rows)
-
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::minivec)) t2;
- count 
--------
-     4
-(1 row)
-
-SELECT COUNT(*) FROM t;
- count 
--------
-     5
-(1 row)
-
-TRUNCATE t;
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
- val 
------
-(0 rows)
-
-DROP TABLE t;
--- inner product
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val minivec_ip_ops) WITH (lists = 1);
-INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <#> '[3,3,3]';
-   val   
----------
- [1,2,4]
- [1,2,3]
- [1,1,1]
- [0,0,0]
-(4 rows)
-
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::minivec)) t2;
- count 
--------
-     4
-(1 row)
-
-DROP TABLE t;
--- cosine
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val minivec_cosine_ops) WITH (lists = 1);
-INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <=> '[3,3,3]';
-   val   
----------
- [1,1,1]
- [1,2,3]
- [1,2,4]
-(3 rows)
-
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
- count 
--------
-     3
-(1 row)
-
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::minivec)) t2;
- count 
--------
-     3
-(1 row)
-
-DROP TABLE t;

test/expected/minivec.out

@@ -1,590 +0,0 @@
-SELECT '[1,2,3]'::minivec;
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '[-1,-2,-3]'::minivec;
-  minivec   
-------------
- [-1,-2,-3]
-(1 row)
-
-SELECT '[1.,2.,3.]'::minivec;
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT ' [ 1,  2 ,    3  ] '::minivec;
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '[1.23456]'::minivec;
- minivec 
----------
- [1.25]
-(1 row)
-
-SELECT '[hello,1]'::minivec;
-ERROR:  invalid input syntax for type minivec: "[hello,1]"
-LINE 1: SELECT '[hello,1]'::minivec;
-               ^
-SELECT '[NaN,1]'::minivec;
-ERROR:  NaN not allowed in minivec
-LINE 1: SELECT '[NaN,1]'::minivec;
-               ^
-SELECT '[Infinity,1]'::minivec;
-ERROR:  infinite value not allowed in minivec
-LINE 1: SELECT '[Infinity,1]'::minivec;
-               ^
-SELECT '[-Infinity,1]'::minivec;
-ERROR:  infinite value not allowed in minivec
-LINE 1: SELECT '[-Infinity,1]'::minivec;
-               ^
-SELECT '[61439,-61439]'::minivec;
-    minivec     
-----------------
- [57344,-57344]
-(1 row)
-
-SELECT '[61440,-61440]'::minivec;
-ERROR:  "61440" is out of range for type minivec
-LINE 1: SELECT '[61440,-61440]'::minivec;
-               ^
-SELECT '[1e-8,-1e-8]'::minivec;
- minivec 
----------
- [0,-0]
-(1 row)
-
-SELECT '[4e38,1]'::minivec;
-ERROR:  "4e38" is out of range for type minivec
-LINE 1: SELECT '[4e38,1]'::minivec;
-               ^
-SELECT '[1e-46,1]'::minivec;
- minivec 
----------
- [0,1]
-(1 row)
-
-SELECT '[1,2,3'::minivec;
-ERROR:  invalid input syntax for type minivec: "[1,2,3"
-LINE 1: SELECT '[1,2,3'::minivec;
-               ^
-SELECT '[1,2,3]9'::minivec;
-ERROR:  invalid input syntax for type minivec: "[1,2,3]9"
-LINE 1: SELECT '[1,2,3]9'::minivec;
-               ^
-DETAIL:  Junk after closing right brace.
-SELECT '1,2,3'::minivec;
-ERROR:  invalid input syntax for type minivec: "1,2,3"
-LINE 1: SELECT '1,2,3'::minivec;
-               ^
-DETAIL:  Vector contents must start with "[".
-SELECT ''::minivec;
-ERROR:  invalid input syntax for type minivec: ""
-LINE 1: SELECT ''::minivec;
-               ^
-DETAIL:  Vector contents must start with "[".
-SELECT '['::minivec;
-ERROR:  invalid input syntax for type minivec: "["
-LINE 1: SELECT '['::minivec;
-               ^
-SELECT '[ '::minivec;
-ERROR:  invalid input syntax for type minivec: "[ "
-LINE 1: SELECT '[ '::minivec;
-               ^
-SELECT '[,'::minivec;
-ERROR:  invalid input syntax for type minivec: "[,"
-LINE 1: SELECT '[,'::minivec;
-               ^
-SELECT '[]'::minivec;
-ERROR:  minivec must have at least 1 dimension
-LINE 1: SELECT '[]'::minivec;
-               ^
-SELECT '[ ]'::minivec;
-ERROR:  minivec must have at least 1 dimension
-LINE 1: SELECT '[ ]'::minivec;
-               ^
-SELECT '[,]'::minivec;
-ERROR:  invalid input syntax for type minivec: "[,]"
-LINE 1: SELECT '[,]'::minivec;
-               ^
-SELECT '[1,]'::minivec;
-ERROR:  invalid input syntax for type minivec: "[1,]"
-LINE 1: SELECT '[1,]'::minivec;
-               ^
-SELECT '[1a]'::minivec;
-ERROR:  invalid input syntax for type minivec: "[1a]"
-LINE 1: SELECT '[1a]'::minivec;
-               ^
-SELECT '[1,,3]'::minivec;
-ERROR:  invalid input syntax for type minivec: "[1,,3]"
-LINE 1: SELECT '[1,,3]'::minivec;
-               ^
-SELECT '[1, ,3]'::minivec;
-ERROR:  invalid input syntax for type minivec: "[1, ,3]"
-LINE 1: SELECT '[1, ,3]'::minivec;
-               ^
-SELECT '[1,2,3]'::minivec(3);
- minivec 
----------
- [1,2,3]
-(1 row)
-
-SELECT '[1,2,3]'::minivec(2);
-ERROR:  expected 2 dimensions, not 3
-SELECT '[1,2,3]'::minivec(3, 2);
-ERROR:  invalid type modifier
-LINE 1: SELECT '[1,2,3]'::minivec(3, 2);
-                          ^
-SELECT '[1,2,3]'::minivec('a');
-ERROR:  invalid input syntax for type integer: "a"
-LINE 1: SELECT '[1,2,3]'::minivec('a');
-                          ^
-SELECT '[1,2,3]'::minivec(0);
-ERROR:  dimensions for type minivec must be at least 1
-LINE 1: SELECT '[1,2,3]'::minivec(0);
-                          ^
-SELECT '[1,2,3]'::minivec(16001);
-ERROR:  dimensions for type minivec cannot exceed 16000
-LINE 1: SELECT '[1,2,3]'::minivec(16001);
-                          ^
-SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::minivec[]);
- unnest  
----------
- [1,2,3]
- [4,5,6]
-(2 rows)
-
-SELECT '{"[1,2,3]"}'::minivec(2)[];
-ERROR:  expected 2 dimensions, not 3
-SELECT '[1,2,3]'::minivec + '[4,5,6]';
- ?column? 
-----------
- [5,7,8]
-(1 row)
-
-SELECT '[61439]'::minivec + '[61439]';
-ERROR:  value out of range: overflow
-SELECT '[1,2]'::minivec + '[3]';
-ERROR:  different minivec dimensions 2 and 1
-SELECT '[1,2,3]'::minivec - '[4,5,6]';
-ERROR:  value out of range: overflow
-SELECT '[-61439]'::minivec - '[61439]';
- ?column? 
-----------
- [-0]
-(1 row)
-
-SELECT '[1,2]'::minivec - '[3]';
-ERROR:  different minivec dimensions 2 and 1
-SELECT '[1,2,3]'::minivec * '[4,5,6]';
- ?column?  
------------
- [4,10,16]
-(1 row)
-
-SELECT '[61439]'::minivec * '[61439]';
-ERROR:  value out of range: overflow
-SELECT '[1e-7]'::minivec * '[1e-7]';
- ?column? 
-----------
- [0]
-(1 row)
-
-SELECT '[1,2]'::minivec * '[3]';
-ERROR:  different minivec dimensions 2 and 1
-SELECT '[1,2,3]'::minivec || '[4,5]';
-  ?column?   
--------------
- [1,2,3,4,5]
-(1 row)
-
-SELECT array_fill(0, ARRAY[16000])::minivec || '[1]';
-ERROR:  minivec cannot have more than 16000 dimensions
-SELECT '[1,2,3]'::minivec < '[1,2,3]';
- ?column? 
-----------
- f
-(1 row)
-
-SELECT '[1,2,3]'::minivec < '[1,2]';
- ?column? 
-----------
- f
-(1 row)
-
-SELECT '[1,2,3]'::minivec <= '[1,2,3]';
- ?column? 
-----------
- t
-(1 row)
-
-SELECT '[1,2,3]'::minivec <= '[1,2]';
- ?column? 
-----------
- f
-(1 row)
-
-SELECT '[1,2,3]'::minivec = '[1,2,3]';
- ?column? 
-----------
- t
-(1 row)
-
-SELECT '[1,2,3]'::minivec = '[1,2]';
- ?column? 
-----------
- f
-(1 row)
-
-SELECT '[1,2,3]'::minivec != '[1,2,3]';
- ?column? 
-----------
- f
-(1 row)
-
-SELECT '[1,2,3]'::minivec != '[1,2]';
- ?column? 
-----------
- t
-(1 row)
-
-SELECT '[1,2,3]'::minivec >= '[1,2,3]';
- ?column? 
-----------
- t
-(1 row)
-
-SELECT '[1,2,3]'::minivec >= '[1,2]';
- ?column? 
-----------
- t
-(1 row)
-
-SELECT '[1,2,3]'::minivec > '[1,2,3]';
- ?column? 
-----------
- f
-(1 row)
-
-SELECT '[1,2,3]'::minivec > '[1,2]';
- ?column? 
-----------
- t
-(1 row)
-
-SELECT minivec_cmp('[1,2,3]', '[1,2,3]');
- minivec_cmp 
--------------
-           0
-(1 row)
-
-SELECT minivec_cmp('[1,2,3]', '[0,0,0]');
- minivec_cmp 
--------------
-           1
-(1 row)
-
-SELECT minivec_cmp('[0,0,0]', '[1,2,3]');
- minivec_cmp 
--------------
-          -1
-(1 row)
-
-SELECT minivec_cmp('[1,2]', '[1,2,3]');
- minivec_cmp 
--------------
-          -1
-(1 row)
-
-SELECT minivec_cmp('[1,2,3]', '[1,2]');
- minivec_cmp 
--------------
-           1
-(1 row)
-
-SELECT minivec_cmp('[1,2]', '[2,3,4]');
- minivec_cmp 
--------------
-          -1
-(1 row)
-
-SELECT minivec_cmp('[2,3]', '[1,2,3]');
- minivec_cmp 
--------------
-           1
-(1 row)
-
-SELECT vector_dims('[1,2,3]'::minivec);
- vector_dims 
--------------
-           3
-(1 row)
-
-SELECT round(l2_norm('[1,1]'::minivec)::numeric, 5);
-  round  
----------
- 1.41421
-(1 row)
-
-SELECT l2_norm('[3,4]'::minivec);
- l2_norm 
----------
-       5
-(1 row)
-
-SELECT l2_norm('[0,1]'::minivec);
- l2_norm 
----------
-       1
-(1 row)
-
-SELECT l2_norm('[0,0]'::minivec);
- l2_norm 
----------
-       0
-(1 row)
-
-SELECT l2_norm('[2]'::minivec);
- l2_norm 
----------
-       2
-(1 row)
-
-SELECT l2_distance('[0,0]'::minivec, '[3,4]');
- l2_distance 
--------------
-           5
-(1 row)
-
-SELECT l2_distance('[0,0]'::minivec, '[0,1]');
- l2_distance 
--------------
-           1
-(1 row)
-
-SELECT l2_distance('[1,2]'::minivec, '[3]');
-ERROR:  different minivec dimensions 2 and 1
-SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,1,1,1,1,1,1,4,5]');
- l2_distance 
--------------
-           5
-(1 row)
-
-SELECT '[0,0]'::minivec <-> '[3,4]';
- ?column? 
-----------
-        5
-(1 row)
-
-SELECT inner_product('[1,2]'::minivec, '[3,4]');
- inner_product 
----------------
-            11
-(1 row)
-
-SELECT inner_product('[1,2]'::minivec, '[3]');
-ERROR:  different minivec dimensions 2 and 1
-SELECT inner_product('[448]'::minivec, '[448]');
- inner_product 
----------------
-        200704
-(1 row)
-
-SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]');
- inner_product 
----------------
-            44
-(1 row)
-
-SELECT '[1,2]'::minivec <#> '[3,4]';
- ?column? 
-----------
-      -11
-(1 row)
-
-SELECT cosine_distance('[1,2]'::minivec, '[2,4]');
- cosine_distance 
------------------
-               0
-(1 row)
-
-SELECT cosine_distance('[1,2]'::minivec, '[0,0]');
- cosine_distance 
------------------
-             NaN
-(1 row)
-
-SELECT cosine_distance('[1,1]'::minivec, '[1,1]');
- cosine_distance 
------------------
-               0
-(1 row)
-
-SELECT cosine_distance('[1,0]'::minivec, '[0,2]');
- cosine_distance 
------------------
-               1
-(1 row)
-
-SELECT cosine_distance('[1,1]'::minivec, '[-1,-1]');
- cosine_distance 
------------------
-               2
-(1 row)
-
-SELECT cosine_distance('[1,2]'::minivec, '[3]');
-ERROR:  different minivec dimensions 2 and 1
-SELECT cosine_distance('[1,1]'::minivec, '[1.1,1.1]');
- cosine_distance 
------------------
-               0
-(1 row)
-
-SELECT cosine_distance('[1,1]'::minivec, '[-1.1,-1.1]');
- cosine_distance 
------------------
-               2
-(1 row)
-
-SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
- cosine_distance 
------------------
-               0
-(1 row)
-
-SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
- cosine_distance 
------------------
-               2
-(1 row)
-
-SELECT '[1,2]'::minivec <=> '[2,4]';
- ?column? 
-----------
-        0
-(1 row)
-
-SELECT l1_distance('[0,0]'::minivec, '[3,4]');
- l1_distance 
--------------
-           7
-(1 row)
-
-SELECT l1_distance('[0,0]'::minivec, '[0,1]');
- l1_distance 
--------------
-           1
-(1 row)
-
-SELECT l1_distance('[1,2]'::minivec, '[3]');
-ERROR:  different minivec dimensions 2 and 1
-SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
- l1_distance 
--------------
-           0
-(1 row)
-
-SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]');
- l1_distance 
--------------
-           7
-(1 row)
-
-SELECT '[0,0]'::minivec <+> '[3,4]';
- ?column? 
-----------
-        7
-(1 row)
-
-SELECT l2_normalize('[3,4]'::minivec);
- l2_normalize 
---------------
- [0.625,0.75]
-(1 row)
-
-SELECT l2_normalize('[3,0]'::minivec);
- l2_normalize 
---------------
- [1,0]
-(1 row)
-
-SELECT l2_normalize('[0,0.1]'::minivec);
- l2_normalize 
---------------
- [0,1]
-(1 row)
-
-SELECT l2_normalize('[0,0]'::minivec);
- l2_normalize 
---------------
- [0,0]
-(1 row)
-
-SELECT l2_normalize('[448]'::minivec);
- l2_normalize 
---------------
- [1]
-(1 row)
-
-SELECT binary_quantize('[1,0,-1]'::minivec);
- binary_quantize 
------------------
- 100
-(1 row)
-
-SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::minivec);
- binary_quantize 
------------------
- 01001110101
-(1 row)
-
-SELECT subvector('[1,2,3,4,5]'::minivec, 1, 3);
- subvector 
------------
- [1,2,3]
-(1 row)
-
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2);
- subvector 
------------
- [3,4]
-(1 row)
-
-SELECT subvector('[1,2,3,4,5]'::minivec, -1, 3);
- subvector 
------------
- [1]
-(1 row)
-
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, 9);
- subvector 
------------
- [3,4,5]
-(1 row)
-
-SELECT subvector('[1,2,3,4,5]'::minivec, 1, 0);
-ERROR:  minivec must have at least 1 dimension
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, -1);
-ERROR:  minivec must have at least 1 dimension
-SELECT subvector('[1,2,3,4,5]'::minivec, -1, 2);
-ERROR:  minivec must have at least 1 dimension
-SELECT subvector('[1,2,3,4,5]'::minivec, 2147483647, 10);
-ERROR:  minivec must have at least 1 dimension
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2147483647);
- subvector 
------------
- [3,4,5]
-(1 row)
-
-SELECT subvector('[1,2,3,4,5]'::minivec, -2147483644, 2147483647);
- subvector 
------------
- [1,2]
-(1 row)
-

test/sql/btree.sql

@@ -22,17 +22,6 @@ SELECT * FROM t ORDER BY val;
 
 DROP TABLE t;
 
--- minivec
-
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t (val);
-
-SELECT * FROM t WHERE val = '[1,2,3]';
-SELECT * FROM t ORDER BY val;
-
-DROP TABLE t;
-
 -- sparsevec
 
 CREATE TABLE t (val sparsevec(3));

test/sql/cast.sql

@@ -38,22 +38,6 @@ SELECT '{1,2,3}'::real[]::halfvec(2);
 SELECT '{65520,-65520}'::real[]::halfvec;
 SELECT '{1e-8,-1e-8}'::real[]::halfvec;
 
-SELECT '[1,2,3]'::vector::minivec;
-SELECT '[1,2,3]'::vector::minivec(3);
-SELECT '[1,2,3]'::vector::minivec(2);
-SELECT '[61440]'::vector::minivec;
-SELECT '[1e-8]'::vector::minivec;
-
-SELECT '[1,2,3]'::minivec::vector;
-SELECT '[1,2,3]'::minivec::vector(3);
-SELECT '[1,2,3]'::minivec::vector(2);
-
-SELECT '{1,2,3}'::real[]::minivec;
-SELECT '{1,2,3}'::real[]::minivec(3);
-SELECT '{1,2,3}'::real[]::minivec(2);
-SELECT '{61440,-61440}'::real[]::minivec;
-SELECT '{1e-8,-1e-8}'::real[]::minivec;
-
 SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
 SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(5);
 SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(4);

test/sql/copy.sql

@@ -28,21 +28,6 @@ SELECT * FROM t2 ORDER BY val;
 DROP TABLE t;
 DROP TABLE t2;
 
--- minivec
-
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-
-CREATE TABLE t2 (val minivec(3));
-
-\copy t TO 'results/minivec.bin' WITH (FORMAT binary)
-\copy t2 FROM 'results/minivec.bin' WITH (FORMAT binary)
-
-SELECT * FROM t2 ORDER BY val;
-
-DROP TABLE t;
-DROP TABLE t2;
-
 -- sparsevec
 
 CREATE TABLE t (val sparsevec(3));

test/sql/ivfflat_minivec.sql

@@ -1,45 +0,0 @@
-SET enable_seqscan = off;
-
--- L2
-
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val minivec_l2_ops) WITH (lists = 1);
-
-INSERT INTO t (val) VALUES ('[1,2,4]');
-
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::minivec)) t2;
-SELECT COUNT(*) FROM t;
-
-TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-DROP TABLE t;
-
--- inner product
-
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val minivec_ip_ops) WITH (lists = 1);
-
-INSERT INTO t (val) VALUES ('[1,2,4]');
-
-SELECT * FROM t ORDER BY val <#> '[3,3,3]';
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::minivec)) t2;
-
-DROP TABLE t;
-
--- cosine
-
-CREATE TABLE t (val minivec(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val minivec_cosine_ops) WITH (lists = 1);
-
-INSERT INTO t (val) VALUES ('[1,2,4]');
-
-SELECT * FROM t ORDER BY val <=> '[3,3,3]';
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::minivec)) t2;
-
-DROP TABLE t;

test/sql/minivec.sql

@@ -1,134 +0,0 @@
-SELECT '[1,2,3]'::minivec;
-SELECT '[-1,-2,-3]'::minivec;
-SELECT '[1.,2.,3.]'::minivec;
-SELECT ' [ 1,  2 ,    3  ] '::minivec;
-SELECT '[1.23456]'::minivec;
-SELECT '[hello,1]'::minivec;
-SELECT '[NaN,1]'::minivec;
-SELECT '[Infinity,1]'::minivec;
-SELECT '[-Infinity,1]'::minivec;
-SELECT '[61439,-61439]'::minivec;
-SELECT '[61440,-61440]'::minivec;
-SELECT '[1e-8,-1e-8]'::minivec;
-SELECT '[4e38,1]'::minivec;
-SELECT '[1e-46,1]'::minivec;
-SELECT '[1,2,3'::minivec;
-SELECT '[1,2,3]9'::minivec;
-SELECT '1,2,3'::minivec;
-SELECT ''::minivec;
-SELECT '['::minivec;
-SELECT '[ '::minivec;
-SELECT '[,'::minivec;
-SELECT '[]'::minivec;
-SELECT '[ ]'::minivec;
-SELECT '[,]'::minivec;
-SELECT '[1,]'::minivec;
-SELECT '[1a]'::minivec;
-SELECT '[1,,3]'::minivec;
-SELECT '[1, ,3]'::minivec;
-
-SELECT '[1,2,3]'::minivec(3);
-SELECT '[1,2,3]'::minivec(2);
-SELECT '[1,2,3]'::minivec(3, 2);
-SELECT '[1,2,3]'::minivec('a');
-SELECT '[1,2,3]'::minivec(0);
-SELECT '[1,2,3]'::minivec(16001);
-
-SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::minivec[]);
-SELECT '{"[1,2,3]"}'::minivec(2)[];
-
-SELECT '[1,2,3]'::minivec + '[4,5,6]';
-SELECT '[61439]'::minivec + '[61439]';
-SELECT '[1,2]'::minivec + '[3]';
-
-SELECT '[1,2,3]'::minivec - '[4,5,6]';
-SELECT '[-61439]'::minivec - '[61439]';
-SELECT '[1,2]'::minivec - '[3]';
-
-SELECT '[1,2,3]'::minivec * '[4,5,6]';
-SELECT '[61439]'::minivec * '[61439]';
-SELECT '[1e-7]'::minivec * '[1e-7]';
-SELECT '[1,2]'::minivec * '[3]';
-
-SELECT '[1,2,3]'::minivec || '[4,5]';
-SELECT array_fill(0, ARRAY[16000])::minivec || '[1]';
-
-SELECT '[1,2,3]'::minivec < '[1,2,3]';
-SELECT '[1,2,3]'::minivec < '[1,2]';
-SELECT '[1,2,3]'::minivec <= '[1,2,3]';
-SELECT '[1,2,3]'::minivec <= '[1,2]';
-SELECT '[1,2,3]'::minivec = '[1,2,3]';
-SELECT '[1,2,3]'::minivec = '[1,2]';
-SELECT '[1,2,3]'::minivec != '[1,2,3]';
-SELECT '[1,2,3]'::minivec != '[1,2]';
-SELECT '[1,2,3]'::minivec >= '[1,2,3]';
-SELECT '[1,2,3]'::minivec >= '[1,2]';
-SELECT '[1,2,3]'::minivec > '[1,2,3]';
-SELECT '[1,2,3]'::minivec > '[1,2]';
-
-SELECT minivec_cmp('[1,2,3]', '[1,2,3]');
-SELECT minivec_cmp('[1,2,3]', '[0,0,0]');
-SELECT minivec_cmp('[0,0,0]', '[1,2,3]');
-SELECT minivec_cmp('[1,2]', '[1,2,3]');
-SELECT minivec_cmp('[1,2,3]', '[1,2]');
-SELECT minivec_cmp('[1,2]', '[2,3,4]');
-SELECT minivec_cmp('[2,3]', '[1,2,3]');
-
-SELECT vector_dims('[1,2,3]'::minivec);
-
-SELECT round(l2_norm('[1,1]'::minivec)::numeric, 5);
-SELECT l2_norm('[3,4]'::minivec);
-SELECT l2_norm('[0,1]'::minivec);
-SELECT l2_norm('[0,0]'::minivec);
-SELECT l2_norm('[2]'::minivec);
-
-SELECT l2_distance('[0,0]'::minivec, '[3,4]');
-SELECT l2_distance('[0,0]'::minivec, '[0,1]');
-SELECT l2_distance('[1,2]'::minivec, '[3]');
-SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,1,1,1,1,1,1,4,5]');
-SELECT '[0,0]'::minivec <-> '[3,4]';
-
-SELECT inner_product('[1,2]'::minivec, '[3,4]');
-SELECT inner_product('[1,2]'::minivec, '[3]');
-SELECT inner_product('[448]'::minivec, '[448]');
-SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]');
-SELECT '[1,2]'::minivec <#> '[3,4]';
-
-SELECT cosine_distance('[1,2]'::minivec, '[2,4]');
-SELECT cosine_distance('[1,2]'::minivec, '[0,0]');
-SELECT cosine_distance('[1,1]'::minivec, '[1,1]');
-SELECT cosine_distance('[1,0]'::minivec, '[0,2]');
-SELECT cosine_distance('[1,1]'::minivec, '[-1,-1]');
-SELECT cosine_distance('[1,2]'::minivec, '[3]');
-SELECT cosine_distance('[1,1]'::minivec, '[1.1,1.1]');
-SELECT cosine_distance('[1,1]'::minivec, '[-1.1,-1.1]');
-SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
-SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
-SELECT '[1,2]'::minivec <=> '[2,4]';
-
-SELECT l1_distance('[0,0]'::minivec, '[3,4]');
-SELECT l1_distance('[0,0]'::minivec, '[0,1]');
-SELECT l1_distance('[1,2]'::minivec, '[3]');
-SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
-SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]');
-SELECT '[0,0]'::minivec <+> '[3,4]';
-
-SELECT l2_normalize('[3,4]'::minivec);
-SELECT l2_normalize('[3,0]'::minivec);
-SELECT l2_normalize('[0,0.1]'::minivec);
-SELECT l2_normalize('[0,0]'::minivec);
-SELECT l2_normalize('[448]'::minivec);
-
-SELECT binary_quantize('[1,0,-1]'::minivec);
-SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::minivec);
-
-SELECT subvector('[1,2,3,4,5]'::minivec, 1, 3);
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2);
-SELECT subvector('[1,2,3,4,5]'::minivec, -1, 3);
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, 9);
-SELECT subvector('[1,2,3,4,5]'::minivec, 1, 0);
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, -1);
-SELECT subvector('[1,2,3,4,5]'::minivec, -1, 2);
-SELECT subvector('[1,2,3,4,5]'::minivec, 2147483647, 10);
-SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2147483647);
-SELECT subvector('[1,2,3,4,5]'::minivec, -2147483644, 2147483647);

test/t/009_ivfflat_filtering.pl

@@ -94,8 +94,7 @@ like($explain, qr/Seq Scan/);
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query';
 ));
-# TODO Do not use index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Seq Scan/);
 
 # Test attribute index
 $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
@@ -110,7 +109,6 @@ $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING ivfflat (v v
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Use partial index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Index Scan using partial_idx/);
 
 done_testing();

test/t/017_hnsw_filtering.pl

@@ -18,9 +18,13 @@ $node->start;
 # Create table and index
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
 $node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);");
+$node->safe_psql("postgres", "CREATE TABLE cat (i int4 PRIMARY KEY, t text, b boolean);");
 $node->safe_psql("postgres",
 	"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;"
 );
+$node->safe_psql("postgres",
+	"INSERT INTO cat SELECT i, 'cat ' || i, i % 5 = 0 FROM generate_series(1, $nc) i;"
+);
 $node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
 $node->safe_psql("postgres", "ANALYZE tst;");
 
@@ -37,8 +41,7 @@ my $c = int(rand() * $nc);
 my $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Do not use index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Seq Scan/);
 
 # Test attribute filtering with few rows removed
 $explain = $node->safe_psql("postgres", qq(
@@ -56,8 +59,7 @@ like($explain, qr/Index Scan using idx/);
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c < 1 ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Do not use index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Seq Scan/);
 
 # Test attribute filtering with few rows removed like
 $explain = $node->safe_psql("postgres", qq(
@@ -96,13 +98,25 @@ $explain = $node->safe_psql("postgres", qq(
 ));
 like($explain, qr/Seq Scan/);
 
+# Test join
+$explain = $node->safe_psql("postgres", qq(
+	EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c ORDER BY v <-> '$query' LIMIT $limit;
+));
+like($explain, qr/Index Scan using idx/);
+
+# Test join with attribute filtering
+$explain = $node->safe_psql("postgres", qq(
+	EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c WHERE cat.b = 't' ORDER BY v <-> '$query' LIMIT $limit;
+));
+like($explain, qr/Index Scan using idx/);
+
 # Test attribute index
 $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Use attribute index
-like($explain, qr/Index Scan using idx/);
+# Use attribute index
+like($explain, qr/Bitmap Index Scan on attribute_idx/);
 
 # Test partial index
 $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);");

test/t/033_comparison.pl

@@ -40,10 +40,6 @@ for (1 .. 50)
 	$actual = $node->safe_psql("postgres", "SELECT halfvec_cmp(v::halfvec, '$query'::real[]::halfvec) FROM tst");
 	is($expected, $actual);
 
-	# Test minivec
-	$actual = $node->safe_psql("postgres", "SELECT minivec_cmp(v::minivec, '$query'::real[]::minivec) FROM tst");
-	is($expected, $actual);
-
 	# Test sparsevec
 	$actual = $node->safe_psql("postgres", "SELECT sparsevec_cmp(v::vector::sparsevec, '$query'::real[]::vector::sparsevec) FROM tst");
 	is($expected, $actual);

test/t/034_distance_functions.pl

@@ -45,10 +45,6 @@ for my $function (@functions)
 		my $actual = $node->safe_psql("postgres", "SELECT $function(v::halfvec, '$query'::vector::halfvec) FROM tst");
 		is($expected, $actual, "halfvec $function");
 
-		# Test minivec
-		$actual = $node->safe_psql("postgres", "SELECT $function(v::minivec, '$query'::vector::minivec) FROM tst");
-		is($expected, $actual, "minivec $function");
-
 		# Test sparsevec
 		$actual = $node->safe_psql("postgres", "SELECT $function(v::sparsevec, '$query'::vector::sparsevec) FROM tst");
 		is($expected, $actual, "sparsevec $function");

test/t/037_inputs.pl

@@ -12,8 +12,8 @@ $node->start;
 # Create extension
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
 
-my @types = ("vector", "halfvec", "minivec", "sparsevec");
-my @inputs = ("[1.23,4.56,7.89]", "[1.23,4.56,7.89]", "[1.23,4.56,7.89]", "{1:1.23,2:4.56,3:7.89}/3");
+my @types = ("vector", "halfvec", "sparsevec");
+my @inputs = ("[1.23,4.56,7.89]", "[1.23,4.56,7.89]", "{1:1.23,2:4.56,3:7.89}/3");
 my @subs = (" ", " ", ",", ":", "-", "1", "9", "\0", "2147483648", "-2147483649");
 
 for my $i (0 .. $#types)

test/t/039_hnsw_cost.pl

@@ -0,0 +1,60 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my @dims = (384, 1536);
+my $limit = 10;
+
+# Initialize node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+
+for my $dim (@dims)
+{
+	my $array_sql = join(",", ('random()') x $dim);
+
+	# Create table and index
+	$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
+	$node->safe_psql("postgres",
+		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 2000) i;"
+	);
+	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
+	$node->safe_psql("postgres", "ANALYZE tst;");
+
+	# Generate query
+	my @r = ();
+	for (1 .. $dim)
+	{
+		push(@r, rand());
+	}
+	my $query = "[" . join(",", @r) . "]";
+
+	my $explain = $node->safe_psql("postgres", qq(
+		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx/);
+
+	# 3x the rows are needed for distance filters
+	# since the planner uses DEFAULT_INEQ_SEL for the selectivity (should be 1)
+	# Recreate index for performance
+	$node->safe_psql("postgres", "DROP INDEX idx;");
+	$node->safe_psql("postgres",
+		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(2001, 6000) i;"
+	);
+	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
+	$node->safe_psql("postgres", "ANALYZE tst;");
+
+	$explain = $node->safe_psql("postgres", qq(
+		EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx/);
+
+	$node->safe_psql("postgres", "DROP TABLE tst;");
+}
+
+done_testing();

test/t/039_hnsw_minivec_build_recall.pl

@@ -1,132 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $node;
-my @queries = ();
-my @expected;
-my $limit = 20;
-my $dim = 10;
-my $array_sql = join(",", ('2 * random() * random()') x $dim);
-
-sub test_recall
-{
-	my ($min, $operator) = @_;
-	my $correct = 0;
-	my $total = 0;
-
-	my $explain = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan/);
-
-	for my $i (0 .. $#queries)
-	{
-		my $actual = $node->safe_psql("postgres", qq(
-			SET enable_seqscan = off;
-			SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
-		));
-		my @actual_ids = split("\n", $actual);
-		my %actual_set = map { $_ => 1 } @actual_ids;
-
-		my @expected_ids = split("\n", $expected[$i]);
-
-		foreach (@expected_ids)
-		{
-			if (exists($actual_set{$_}))
-			{
-				$correct++;
-			}
-			$total++;
-		}
-	}
-
-	cmp_ok($correct / $total, ">=", $min, $operator);
-}
-
-# Initialize node
-$node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v minivec($dim));");
-$node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
-);
-
-# Generate queries
-for (1 .. 20)
-{
-	my @r = ();
-	for (1 .. $dim)
-	{
-		push(@r, rand());
-	}
-	push(@queries, "[" . join(",", @r) . "]");
-}
-
-# Check each index type
-my @operators = ("<->", "<#>", "<=>", "<+>");
-my @opclasses = ("minivec_l2_ops", "minivec_ip_ops", "minivec_cosine_ops", "minivec_l1_ops");
-
-for my $i (0 .. $#operators)
-{
-	my $operator = $operators[$i];
-	my $opclass = $opclasses[$i];
-
-	# Get exact results
-	@expected = ();
-	foreach (@queries)
-	{
-		my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
-		push(@expected, $res);
-	}
-
-	# Build index serially
-	$node->safe_psql("postgres", qq(
-		SET max_parallel_maintenance_workers = 0;
-		CREATE INDEX idx ON tst USING hnsw (v $opclass);
-	));
-
-	# Test approximate results
-	my $min = 0.98;
-	test_recall($min, $operator);
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-
-	# Build index in parallel in memory
-	my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
-		SET client_min_messages = DEBUG;
-		SET min_parallel_table_scan_size = 1;
-		CREATE INDEX idx ON tst USING hnsw (v $opclass);
-	));
-	is($ret, 0, $stderr);
-	like($stderr, qr/using \d+ parallel workers/);
-
-	# Test approximate results
-	test_recall($min, $operator);
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-
-	# Build index in parallel on disk
-	# Set parallel_workers on table to use workers with low maintenance_work_mem
-	($ret, $stdout, $stderr) = $node->psql("postgres", qq(
-		ALTER TABLE tst SET (parallel_workers = 2);
-		SET client_min_messages = DEBUG;
-		SET maintenance_work_mem = '4MB';
-		CREATE INDEX idx ON tst USING hnsw (v $opclass);
-		ALTER TABLE tst RESET (parallel_workers);
-	));
-	is($ret, 0, $stderr);
-	like($stderr, qr/using \d+ parallel workers/);
-	like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-}
-
-done_testing();

test/t/040_hnsw_minivec_insert_recall.pl

@@ -1,113 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $node;
-my @queries = ();
-my @expected;
-my $limit = 20;
-my $dim = 10;
-my $array_sql = join(",", ('2 * random() * random()') x $dim);
-
-sub test_recall
-{
-	my ($min, $operator) = @_;
-	my $correct = 0;
-	my $total = 0;
-
-	my $explain = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan/);
-
-	for my $i (0 .. $#queries)
-	{
-		my $actual = $node->safe_psql("postgres", qq(
-			SET enable_seqscan = off;
-			SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
-		));
-		my @actual_ids = split("\n", $actual);
-		my %actual_set = map { $_ => 1 } @actual_ids;
-
-		my @expected_ids = split("\n", $expected[$i]);
-
-		foreach (@expected_ids)
-		{
-			if (exists($actual_set{$_}))
-			{
-				$correct++;
-			}
-			$total++;
-		}
-	}
-
-	cmp_ok($correct / $total, ">=", $min, $operator);
-}
-
-# Initialize node
-$node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v minivec($dim));");
-
-# Generate queries
-for (1 .. 20)
-{
-	my @r = ();
-	for (1 .. $dim)
-	{
-		push(@r, rand());
-	}
-	push(@queries, "[" . join(",", @r) . "]");
-}
-
-# Check each index type
-my @operators = ("<->", "<#>", "<=>", "<+>");
-my @opclasses = ("minivec_l2_ops", "minivec_ip_ops", "minivec_cosine_ops", "minivec_l1_ops");
-
-for my $i (0 .. $#operators)
-{
-	my $operator = $operators[$i];
-	my $opclass = $opclasses[$i];
-
-	# Add index
-	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v $opclass);");
-
-	# Use concurrent inserts
-	$node->pgbench(
-		"--no-vacuum --client=10 --transactions=1000",
-		0,
-		[qr{actually processed}],
-		[qr{^$}],
-		"concurrent INSERTs",
-		{
-			"040_hnsw_minivec_insert_recall_$opclass" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);"
-		}
-	);
-
-	# Get exact results
-	@expected = ();
-	foreach (@queries)
-	{
-		my $res = $node->safe_psql("postgres", qq(
-			SET enable_indexscan = off;
-			SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;
-		));
-		push(@expected, $res);
-	}
-
-	# Test approximate results
-	my $min = 0.98;
-	test_recall($min, $operator);
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-	$node->safe_psql("postgres", "TRUNCATE tst;");
-}
-
-done_testing();

test/t/040_ivfflat_cost.pl

@@ -0,0 +1,50 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my @dims = (384, 1536);
+my $limit = 10;
+
+# Initialize node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+
+for my $dim (@dims)
+{
+	my $array_sql = join(",", ('random()') x $dim);
+
+	# Create table and index
+	$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
+	$node->safe_psql("postgres",
+		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 5000) i;"
+	);
+	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5);");
+	$node->safe_psql("postgres", "ANALYZE tst;");
+
+	# Generate query
+	my @r = ();
+	for (1 .. $dim)
+	{
+		push(@r, rand());
+	}
+	my $query = "[" . join(",", @r) . "]";
+
+	my $explain = $node->safe_psql("postgres", qq(
+		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx/);
+
+	$explain = $node->safe_psql("postgres", qq(
+		EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
+	));
+	like($explain, qr/Index Scan using idx/);
+
+	$node->safe_psql("postgres", "DROP TABLE tst;");
+}
+
+done_testing();

test/t/041_hnsw_filtering.pl

@@ -0,0 +1,109 @@
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node;
+my @queries = ();
+my @cs = ();
+my @expected;
+my $limit = 20;
+my $dim = 3;
+my $array_sql = join(",", ('random()') x $dim);
+my $nc = 50;
+
+sub test_recall
+{
+	my ($min, $operator) = @_;
+	my $correct = 0;
+	my $total = 0;
+
+	my $explain = $node->safe_psql("postgres", qq(
+		SET enable_seqscan = off;
+		EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $cs[0] ORDER BY v $operator '$queries[0]' LIMIT $limit;
+	));
+	like($explain, qr/Index Cond/);
+
+	for my $i (0 .. $#queries)
+	{
+		my $actual = $node->safe_psql("postgres", qq(
+			SET enable_seqscan = off;
+			SELECT i FROM tst WHERE c = $cs[$i] ORDER BY v $operator '$queries[$i]' LIMIT $limit;
+		));
+		my @actual_ids = split("\n", $actual);
+		my %actual_set = map { $_ => 1 } @actual_ids;
+
+		is(scalar(@actual_ids), $limit);
+
+		my @expected_ids = split("\n", $expected[$i]);
+
+		foreach (@expected_ids)
+		{
+			if (exists($actual_set{$_}))
+			{
+				$correct++;
+			}
+			$total++;
+		}
+	}
+
+	cmp_ok($correct / $total, ">=", $min, $operator);
+}
+
+# Initialize node
+$node = PostgreSQL::Test::Cluster->new('node');
+$node->init;
+$node->start;
+
+# Create table
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4);");
+$node->safe_psql("postgres",
+	"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc FROM generate_series(1, 20000) i;"
+);
+
+# Generate queries
+for (1 .. 20)
+{
+	my @r = ();
+	for (1 .. $dim)
+	{
+		push(@r, rand());
+	}
+	push(@queries, "[" . join(",", @r) . "]");
+	push(@cs, int(rand() * $nc));
+}
+
+# Get exact results
+@expected = ();
+for my $i (0 .. $#queries)
+{
+	my $res = $node->safe_psql("postgres", "SELECT i FROM tst WHERE c = $cs[$i] ORDER BY v <-> '$queries[$i]' LIMIT $limit;");
+	push(@expected, $res);
+}
+
+# Add index
+$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops, c);");
+
+# Test recall
+test_recall(0.99, '<->');
+
+# Test vacuum
+$node->safe_psql("postgres", "DELETE FROM tst WHERE c > 5;");
+$node->safe_psql("postgres", "VACUUM tst;");
+
+# Test columns
+my ($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (c);");
+like($stderr, qr/first column must be a vector/);
+
+($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (c, v vector_l2_ops);");
+like($stderr, qr/first column must be a vector/);
+
+($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops, c, c);");
+like($stderr, qr/index cannot have more than two columns/);
+
+($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops, v vector_l2_ops);");
+like($stderr, qr/column 2 cannot be a vector/);
+
+done_testing();

test/t/041_hnsw_minivec_vacuum_recall.pl

@@ -1,97 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $node;
-my @queries = ();
-my @expected;
-my $limit = 20;
-
-sub test_recall
-{
-	my ($min, $ef_search, $test_name) = @_;
-	my $correct = 0;
-	my $total = 0;
-
-	my $explain = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		SET hnsw.ef_search = $ef_search;
-		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$queries[0]' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan/);
-
-	for my $i (0 .. $#queries)
-	{
-		my $actual = $node->safe_psql("postgres", qq(
-			SET enable_seqscan = off;
-			SET hnsw.ef_search = $ef_search;
-			SELECT i FROM tst ORDER BY v <-> '$queries[$i]' LIMIT $limit;
-		));
-		my @actual_ids = split("\n", $actual);
-		my %actual_set = map { $_ => 1 } @actual_ids;
-
-		my @expected_ids = split("\n", $expected[$i]);
-
-		foreach (@expected_ids)
-		{
-			if (exists($actual_set{$_}))
-			{
-				$correct++;
-			}
-			$total++;
-		}
-	}
-
-	cmp_ok($correct / $total, ">=", $min, $test_name);
-}
-
-# Initialize node
-$node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v minivec(3));");
-$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
-$node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
-);
-
-# Add index
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v minivec_l2_ops) WITH (m = 4, ef_construction = 8);");
-
-# Delete data
-$node->safe_psql("postgres", "DELETE FROM tst WHERE i > 2500;");
-
-# Generate queries
-for (1 .. 20)
-{
-	my $r1 = rand();
-	my $r2 = rand();
-	my $r3 = rand();
-	push(@queries, "[$r1,$r2,$r3]");
-}
-
-# Get exact results
-@expected = ();
-foreach (@queries)
-{
-	my $res = $node->safe_psql("postgres", qq(
-		SET enable_indexscan = off;
-		SELECT i FROM tst ORDER BY v <-> '$_' LIMIT $limit;
-	));
-	push(@expected, $res);
-}
-
-test_recall(0.18, $limit, "before vacuum");
-test_recall(0.93, 100, "before vacuum");
-
-# TODO Test concurrent inserts with vacuum
-$node->safe_psql("postgres", "VACUUM tst;");
-
-test_recall(0.95, $limit, "after vacuum");
-
-done_testing();

test/t/042_hnsw_minivec_duplicates.pl

@@ -1,58 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (v minivec(3));");
-
-sub insert_vectors
-{
-	for my $i (1 .. 20)
-	{
-		$node->safe_psql("postgres", "INSERT INTO tst VALUES ('[1,1,1]');");
-	}
-}
-
-sub test_duplicates
-{
-	my $res = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		SET hnsw.ef_search = 1;
-		SELECT COUNT(*) FROM (SELECT * FROM tst ORDER BY v <-> '[1,1,1]') t;
-	));
-	is($res, 10);
-}
-
-# Test duplicates with build
-insert_vectors();
-$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v minivec_l2_ops);");
-test_duplicates();
-
-# Reset
-$node->safe_psql("postgres", "TRUNCATE tst;");
-
-# Test duplicates with inserts
-insert_vectors();
-test_duplicates();
-
-# Test fallback path for inserts
-$node->pgbench(
-	"--no-vacuum --client=5 --transactions=100",
-	0,
-	[qr{actually processed}],
-	[qr{^$}],
-	"concurrent INSERTs",
-	{
-		"042_hnsw_minivec_duplicates" => "INSERT INTO tst VALUES ('[1,1,1]');"
-	}
-);
-
-done_testing();

test/t/043_ivfflat_minivec_build_recall.pl

@@ -1,154 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $node;
-my @queries = ();
-my @expected;
-my $limit = 20;
-my $dim = 10;
-my $array_sql = join(",", ('random()') x $dim);
-
-sub test_recall
-{
-	my ($probes, $min, $operator) = @_;
-	my $correct = 0;
-	my $total = 0;
-
-	my $explain = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		SET ivfflat.probes = $probes;
-		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan using idx on tst/);
-
-	for my $i (0 .. $#queries)
-	{
-		my $actual = $node->safe_psql("postgres", qq(
-			SET enable_seqscan = off;
-			SET ivfflat.probes = $probes;
-			SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
-		));
-		my @actual_ids = split("\n", $actual);
-
-		my @expected_ids = split("\n", $expected[$i]);
-		my %expected_set = map { $_ => 1 } @expected_ids;
-
-		foreach (@actual_ids)
-		{
-			if (exists($expected_set{$_}))
-			{
-				$correct++;
-			}
-		}
-
-		$total += $limit;
-	}
-
-	cmp_ok($correct / $total, ">=", $min, $operator);
-}
-
-# Initialize node
-$node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v minivec($dim));");
-$node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
-);
-
-# Generate queries
-for (1 .. 20)
-{
-	my @r = ();
-	for (1 .. $dim)
-	{
-		push(@r, rand());
-	}
-	push(@queries, "[" . join(",", @r) . "]");
-}
-
-# Check each index type
-my @operators = ("<->", "<#>", "<=>");
-my @opclasses = ("minivec_l2_ops", "minivec_ip_ops", "minivec_cosine_ops");
-
-for my $i (0 .. $#operators)
-{
-	my $operator = $operators[$i];
-	my $opclass = $opclasses[$i];
-
-	# Get exact results
-	@expected = ();
-	foreach (@queries)
-	{
-		my $res = $node->safe_psql("postgres", qq(
-			WITH top AS (
-				SELECT v $operator '$_' AS distance FROM tst ORDER BY distance LIMIT $limit
-			)
-			SELECT i FROM tst WHERE (v $operator '$_') <= (SELECT MAX(distance) FROM top)
-		));
-		push(@expected, $res);
-	}
-
-	# Build index serially
-	$node->safe_psql("postgres", qq(
-		SET max_parallel_maintenance_workers = 0;
-		CREATE INDEX idx ON tst USING ivfflat (v $opclass);
-	));
-
-	# Test approximate results
-	if ($operator ne "<#>")
-	{
-		# TODO Fix test (uniform random vectors all have similar inner product)
-		test_recall(1, 0.33, $operator);
-		test_recall(10, 0.93, $operator);
-	}
-
-	# Test probes equals lists
-	if ($operator eq "<=>")
-	{
-		test_recall(100, 0.98, $operator);
-	}
-	else
-	{
-		test_recall(100, 1.00, $operator);
-	}
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-
-	# Build index in parallel
-	my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
-		SET client_min_messages = DEBUG;
-		SET min_parallel_table_scan_size = 1;
-		CREATE INDEX idx ON tst USING ivfflat (v $opclass);
-	));
-	is($ret, 0, $stderr);
-	like($stderr, qr/using \d+ parallel workers/);
-
-	# Test approximate results
-	if ($operator ne "<#>")
-	{
-		# TODO Fix test (uniform random vectors all have similar inner product)
-		test_recall(1, 0.33, $operator);
-		test_recall(10, 0.93, $operator);
-	}
-
-	# Test probes equals lists
-	if ($operator eq "<=>")
-	{
-		test_recall(100, 0.98, $operator);
-	}
-	else
-	{
-		test_recall(100, 1.00, $operator);
-	}
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-}
-
-done_testing();