Removed duplicated macros [skip ci]

.dockerignore

@@ -0,0 +1,8 @@
+/.git/
+/dist/
+/results/
+/tmp_check/
+/sql/vector--?.?.?.sql
+regression.*
+*.o
+*.so

.github/workflows/build.yml

@@ -8,31 +8,27 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - postgres: 20
-            os: ubuntu-26.04
-          - postgres: 19
-            os: ubuntu-26.04
-          - postgres: 18
-            os: ubuntu-26.04-arm
+          # - postgres: 18
+          #   os: ubuntu-24.04
           - postgres: 17
             os: ubuntu-24.04
           - postgres: 16
-            os: ubuntu-24.04-arm
+            os: ubuntu-22.04
           - postgres: 15
             os: ubuntu-22.04
           - postgres: 14
-            os: ubuntu-22.04-arm
+            os: ubuntu-20.04
           - postgres: 13
-            os: ubuntu-22.04
+            os: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: ${{ matrix.postgres }}
           dev-files: true
       - run: make
         env:
-          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
+          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
       - run: |
           export PG_CONFIG=`which pg_config`
           sudo --preserve-env=PG_CONFIG make install
@@ -50,18 +46,18 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - postgres: 18
-            os: macos-26
+          - postgres: 16
+            os: macos-14
           - postgres: 14
-            os: macos-15-intel
+            os: macos-12
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: ${{ matrix.postgres }}
       - run: make
         env:
-          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
+          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
       - run: make install
       - run: make installcheck
       - if: ${{ failure() }}
@@ -74,35 +70,26 @@ jobs:
           tar xf $TAG.tar.gz
           mv postgres-$TAG postgres
         env:
-          TAG: ${{ matrix.postgres == 18 && 'REL_18_2' || 'REL_14_21' }}
+          TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
       - run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
         env:
           PERL5LIB: /Users/runner/perl5/lib/perl5
-      - run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make
+      - run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
         env:
-          LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }}
           PG_CFLAGS: -DUSE_ASSERT_CHECKING
   windows:
-    runs-on: ${{ matrix.os }}
+    runs-on: windows-latest
     if: ${{ !startsWith(github.ref_name, 'mac') }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - postgres: 17
-            os: windows-2025
-          - postgres: 14
-            os: windows-2022
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres@v1
         with:
-          postgres-version: ${{ matrix.postgres }}
+          postgres-version: 14
       - run: |
-          call "C:\Program Files\Microsoft Visual Studio\${{ matrix.os == 'windows-2025' && 18 || 2022 }}\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
+          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
           nmake /NOLOGO /F Makefile.win && ^
           nmake /NOLOGO /F Makefile.win install && ^
-          nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
+          nmake /NOLOGO /F Makefile.win installcheck && ^
           nmake /NOLOGO /F Makefile.win clean && ^
           nmake /NOLOGO /F Makefile.win uninstall
         shell: cmd
@@ -135,10 +122,10 @@ jobs:
     if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres-valgrind@v1
         with:
-          postgres-version: 18
+          postgres-version: 16
           check-ub: yes
       - run: make OPTFLAGS=""
       - run: sudo --preserve-env=PG_CONFIG make install

CHANGELOG.md

@@ -1,36 +1,10 @@
-## 0.8.5 (2026-07-08)
-
-- Reduced memory usage for small tables for IVFFlat index builds
-
-## 0.8.4 (2026-06-30)
-
-- Fixed `hnsw graph not repaired` error with HNSW vacuuming
-- Fixed possible error with inserts during HNSW vacuuming
-- Fixed memory exceeding `maintenance_work_mem` with IVFFlat index builds
-
-## 0.8.3 (2026-06-17)
-
-- Fixed possible index corruption with HNSW vacuuming
-- Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18
-
-## 0.8.2 (2026-02-25)
-
-- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)
-- Improved `install` target on Windows
-- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
-
-## 0.8.1 (2025-09-04)
-
-- Added support for Postgres 18 rc1
-- Improved performance of `binary_quantize` function
-
-## 0.8.0 (2024-10-30)
+## 0.8.0 (unreleased)
 
 - Added support for iterative index scans
 - Added casts for arrays to `sparsevec`
-- Improved cost estimation for better index selection when filtering
-- Improved performance of HNSW index scans
+- Improved cost estimation
 - Improved performance of HNSW inserts and on-disk index builds
+- Reduced memory usage for HNSW index scans
 - Dropped support for Postgres 12
 
 ## 0.7.4 (2024-08-05)

Dockerfile

@@ -1,11 +1,8 @@
-# syntax=docker/dockerfile:1
-
 ARG PG_MAJOR=17
-ARG DEBIAN_CODENAME=bookworm
-FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
+FROM postgres:$PG_MAJOR
 ARG PG_MAJOR
 
-ADD https://github.com/pgvector/pgvector.git#v0.8.5 /tmp/pgvector
+COPY . /tmp/pgvector
 
 RUN apt-get update && \
 		apt-mark hold locales && \

LICENSE

@@ -1,4 +1,4 @@
-Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
 
 Portions Copyright (c) 1994, The Regents of the University of California
 

META.json

@@ -2,17 +2,17 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.8.5",
+	"version": "0.7.4",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
 	"license": {
-		"PostgreSQL": "https://www.postgresql.org/about/licence"
+		"PostgreSQL": "http://www.postgresql.org/about/licence"
 	},
 	"prereqs": {
 		"runtime": {
 			"requires": {
-				"PostgreSQL": "13.0.0"
+				"PostgreSQL": "12.0.0"
 			}
 		}
 	},
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.8.5",
+			"version": "0.7.4",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},
@@ -38,7 +38,7 @@
 	"generated_by": "Andrew Kane",
 	"meta-spec": {
 		"version": "1.0.0",
-		"url": "https://pgxn.org/meta/spec.txt"
+		"url": "http://pgxn.org/meta/spec.txt"
 	},
 	"tags": [
 		"vectors",

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.8.5
+EXTVERSION = 0.7.4
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*--*.sql)
@@ -27,11 +27,6 @@ ifneq ($(filter ppc64%, $(shell uname -m)), )
 	OPTFLAGS =
 endif
 
-# RISC-V64 doesn't support -march=native
-ifeq ($(shell uname -m), riscv64)
-	OPTFLAGS =
-endif
-
 # For auto-vectorization:
 # - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
 # - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -81,9 +76,4 @@ docker:
 .PHONY: docker-release
 
 docker-release:
-	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm .
-
-.PHONY: docker-release-trixie
-
-docker-release-trixie:
-	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie .
+	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.8.5
+EXTVERSION = 0.7.4
 
 DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
 OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
@@ -31,9 +31,6 @@ LIBDIR = $(PGROOT)\lib
 PKGLIBDIR = $(PGROOT)\lib
 SHAREDIR = $(PGROOT)\share
 
-# Use $(PGROOT)\bin\pg_regress for Postgres < 17
-PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress
-
 CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
 
 CFLAGS = $(CFLAGS) $(PG_CFLAGS)
@@ -57,11 +54,11 @@ install: all
 	copy $(SHLIB) "$(PKGLIBDIR)"
 	copy $(EXTENSION).control "$(SHAREDIR)\extension"
 	copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
-	if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
+	mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
 	for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
 
 installcheck:
-	"$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
+	"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
 
 uninstall:
 	del /f "$(PKGLIBDIR)\$(SHLIB)"

README.md

@@ -11,19 +11,17 @@ Store your vectors with the rest of your data. Supports:
 
 Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
 
-Have a lot of vectors? Use [quantization](#scaling) to scale
-
 [![Build Status](https://github.com/pgvector/pgvector/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector/actions)
 
 ## Installation
 
 ### Linux and Mac
 
-Compile and install the extension (supports Postgres 13+)
+Compile and install the extension (supports Postgres 12+)
 
 ```sh
 cd /tmp
-git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -31,21 +29,31 @@ make install # may need sudo
 
 See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
 
-You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), [APK](#apk), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
+You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
 
 ### Windows
 
-Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build:
+Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
 
 ```cmd
-set "PGROOT=C:\Program Files\PostgreSQL\18"
+call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+```
+
+Note: The exact path will vary depending on your Visual Studio version and edition
+
+Then use `nmake` to build:
+
+```cmd
+set "PGROOT=C:\Program Files\PostgreSQL\16"
 cd %TEMP%
-git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
 ```
 
+Note: Postgres 17 is not supported yet due to an upstream issue
+
 See the [installation notes](#installation-notes---windows) if you run into issues
 
 You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -76,7 +84,7 @@ Get the nearest neighbors by L2 distance
 SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
 ```
 
-Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`)
+Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0)
 
 Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
 
@@ -140,9 +148,9 @@ Supported distance functions are:
 - `<->` - L2 distance
 - `<#>` - (negative) inner product
 - `<=>` - cosine distance
-- `<+>` - L1 distance
-- `<~>` - Hamming distance (binary vectors)
-- `<%>` - Jaccard distance (binary vectors)
+- `<+>` - L1 distance (added in 0.7.0)
+- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
+- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
 
 Get the nearest neighbors to a row
 
@@ -229,19 +237,19 @@ Cosine distance
 CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
 ```
 
-L1 distance
+L1 distance - added in 0.7.0
 
 ```sql
 CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
 ```
 
-Hamming distance
+Hamming distance - added in 0.7.0
 
 ```sql
 CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
 ```
 
-Jaccard distance
+Jaccard distance - added in 0.7.0
 
 ```sql
 CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
@@ -250,9 +258,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
 Supported types are:
 
 - `vector` - up to 2,000 dimensions
-- `halfvec` - up to 4,000 dimensions
-- `bit` - up to 64,000 dimensions
-- `sparsevec` - up to 1,000 non-zero elements
+- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
+- `bit` - up to 64,000 dimensions (added in 0.7.0)
+- `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0)
 
 ### Index Options
 
@@ -306,21 +314,17 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t
 
 Like other index types, it’s faster to create an index after loading your initial data
 
-You can also speed up index creation by increasing the number of parallel workers (2 by default)
+Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
 
 ```sql
 SET max_parallel_maintenance_workers = 7; -- plus leader
 ```
 
-For a large number of workers, you may need to increase `max_parallel_workers` (8 by default)
-
-The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
-
-Use [binary quantization](#binary-quantization) for faster build times at scale
+For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
 
 ### Indexing Progress
 
-Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
+Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
 
 ```sql
 SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -363,7 +367,7 @@ Cosine distance
 CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
 ```
 
-Hamming distance
+Hamming distance - added in 0.7.0
 
 ```sql
 CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
@@ -372,8 +376,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
 Supported types are:
 
 - `vector` - up to 2,000 dimensions
-- `halfvec` - up to 4,000 dimensions
-- `bit` - up to 64,000 dimensions
+- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
+- `bit` - up to 64,000 dimensions (added in 0.7.0)
 
 ### Query Options
 
@@ -406,7 +410,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
 
 ### Indexing Progress
 
-Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
+Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
 
 ```sql
 SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -423,109 +427,92 @@ Note: `%` is only populated during the `loading tuples` phase
 
 ## Filtering
 
-There are a few ways to index nearest neighbor queries with a `WHERE` clause.
+There are a few ways to index nearest neighbor queries with a `WHERE` clause
 
 ```sql
 SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
 ```
 
-A good place to start is creating an index on the filter column. This can provide fast, exact nearest neighbor search in many cases. Postgres has a number of [index types](https://www.postgresql.org/docs/current/indexes-types.html) for this: B-tree (default), hash, GiST, SP-GiST, GIN, and BRIN.
+Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
 
 ```sql
 CREATE INDEX ON items (category_id);
 ```
 
-For multiple columns, consider a [multicolumn index](https://www.postgresql.org/docs/current/indexes-multicolumn.html).
-
-```sql
-CREATE INDEX ON items (location_id, category_id);
-```
-
-Exact indexes work well for conditions that match a low percentage of rows. Otherwise, [approximate indexes](#indexing) can work better.
-
-```sql
-CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
-```
-
-With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
-
-```sql
-SET hnsw.iterative_scan = strict_order;
-```
-
-If filtering by only a few distinct values, consider [partial indexing](https://www.postgresql.org/docs/current/indexes-partial.html).
+Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
 
 ```sql
 CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WHERE (category_id = 123);
 ```
 
-If filtering by many different values, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html).
+Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
 
 ```sql
 CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
 ```
 
-## Iterative Index Scans
+## Iterative Search [unreleased]
 
-With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
+*Added in 0.8.0*
 
-Iterative scans can use strict or relaxed ordering.
+With approximate indexes, you can end up with less results than expected due to filtering conditions in the query.
 
-Strict ensures results are in the exact order by distance
+Starting with 0.8.0, you can enable iterative search. If too few results from the initial index scan match the query filters, it will resume scanning until enough results are found. This can significantly improve recall (at the cost of speed).
 
-```sql
-SET hnsw.iterative_scan = strict_order;
+```tsql
+SET hnsw.streaming = on;
+-- or
+SET ivfflat.streaming = on;
 ```
 
-Relaxed allows results to be slightly out of order by distance, but provides better recall
+However, there are some important caveats.
+
+### Iterative Caveats
+
+With iterative search, it’s possible for rows to be slightly out of order by distance. For strict ordering, use:
 
 ```sql
-SET hnsw.iterative_scan = relaxed_order;
-# or
-SET ivfflat.iterative_scan = relaxed_order;
+WITH approx_order AS MATERIALIZED (
+    SELECT *, embedding <-> '[1,2,3]' AS distance FROM items WHERE ... ORDER BY distance LIMIT 5
+) SELECT * FROM approx_order ORDER BY distance;
 ```
 
-With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-CTE-MATERIALIZATION) to get strict ordering
+For distance filters, use a CTE and place the filter outside it.
 
 ```sql
-WITH relaxed_results AS MATERIALIZED (
-    SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
-) SELECT * FROM relaxed_results ORDER BY distance + 0;
+WITH approx_order AS MATERIALIZED (
+    SELECT *, embedding <-> '[1,2,3]' AS distance FROM items WHERE ... ORDER BY distance LIMIT 5
+) SELECT * FROM approx_order WHERE distance < 0.1 ORDER BY distance;
 ```
 
-Note: `+ 0` is needed for Postgres 17+
+### Iterative Options
 
-For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
-
-```sql
-WITH nearest_results AS MATERIALIZED (
-    SELECT id, embedding <-> '[1,2,3]' AS distance FROM items ORDER BY distance LIMIT 5
-) SELECT * FROM nearest_results WHERE distance < 5 ORDER BY distance;
-```
-
-Note: Place any other filters inside the CTE
-
-### Iterative Scan Options
-
-Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends.
+Since scanning a large portion of the index is expensive, there are options to control when the scan ends.
 
 #### HNSW
 
-Specify the max number of tuples to visit (20,000 by default)
+Specify the max number of additional tuples visited
 
 ```sql
-SET hnsw.max_scan_tuples = 20000;
+SET hnsw.ef_stream = 10000;
 ```
 
-Note: This is approximate and does not affect the initial scan
-
-Specify the max amount of memory to use, as a multiple of `work_mem` (1 by default)
+The scan will also end if reaches `work_mem`. You can see when this happens by enabling debug messages.
 
 ```sql
-SET hnsw.scan_mem_multiplier = 2;
+SET client_min_messages = debug1;
 ```
 
-Note: Try increasing this if increasing `hnsw.max_scan_tuples` does not improve recall
+```text
+DEBUG:  hnsw index scan exceeded work_mem after 10000 tuples
+HINT:  Increase work_mem to scan more tuples.
+```
+
+If the server has enough memory, you can adjust this with:
+
+```sql
+SET work_mem = '8MB';
+```
 
 #### IVFFlat
 
@@ -535,10 +522,10 @@ Specify the max number of probes
 SET ivfflat.max_probes = 100;
 ```
 
-Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used
-
 ## Half-Precision Vectors
 
+*Added in 0.7.0*
+
 Use the `halfvec` type to store half-precision vectors
 
 ```sql
@@ -547,6 +534,8 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
 
 ## Half-Precision Indexing
 
+*Added in 0.7.0*
+
 Index vectors at half precision for smaller indexes
 
 ```sql
@@ -568,16 +557,24 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
 INSERT INTO items (embedding) VALUES ('000'), ('111');
 ```
 
-Get the nearest neighbors by Hamming distance
+Get the nearest neighbors by Hamming distance (added in 0.7.0)
 
 ```sql
 SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
 ```
 
+Or (before 0.7.0)
+
+```sql
+SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
+```
+
 Also supports Jaccard distance (`<%>`)
 
 ## Binary Quantization
 
+*Added in 0.7.0*
+
 Use expression indexing for binary quantization
 
 ```sql
@@ -600,6 +597,8 @@ SELECT * FROM (
 
 ## Sparse Vectors
 
+*Added in 0.7.0*
+
 Use the `sparsevec` type to store sparse vectors
 
 ```sql
@@ -633,6 +632,8 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
 
 ## Indexing Subvectors
 
+*Added in 0.7.0*
+
 Use expression indexing to index subvectors
 
 ```sql
@@ -671,10 +672,6 @@ SHOW shared_buffers;
 
 Be sure to restart Postgres for changes to take effect.
 
-### Storing
-
-Use the `halfvec` type instead of `vector` for a smaller working set.
-
 ### Loading
 
 Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
@@ -689,8 +686,6 @@ Add any indexes *after* loading the initial data for best performance.
 
 See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
 
-Use [binary quantization](#binary-quantization) for smaller indexes and faster build times at scale.
-
 In production environments, create indexes concurrently to avoid blocking writes.
 
 ```sql
@@ -699,10 +694,10 @@ CREATE INDEX CONCURRENTLY ...
 
 ### Querying
 
-Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance.
+Use `EXPLAIN ANALYZE` to debug performance.
 
 ```sql
-EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
+EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
 ```
 
 #### Exact Search
@@ -721,8 +716,6 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
 
 #### Approximate Search
 
-Use [binary quantization](#binary-quantization) with re-ranking to keep indexes in-memory at scale.
-
 To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
 
 ```sql
@@ -738,20 +731,23 @@ REINDEX INDEX CONCURRENTLY index_name;
 VACUUM table_name;
 ```
 
-## Scaling
-
-For a smaller working set:
-
-1. Use the `halfvec` type instead of `vector` for tables
-2. Use [binary quantization](#binary-quantization) for indexes (with re-ranking for search)
-
-Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
-
-Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus), [PgDog](https://github.com/pgdogdev/pgdog), or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
-
 ## Monitoring
 
-Use existing tools like [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) or [PgHero](https://github.com/ankane/pghero) to monitor performance.
+Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
+
+```sql
+CREATE EXTENSION pg_stat_statements;
+```
+
+Get the most time-consuming queries with:
+
+```sql
+SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
+    ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
+    FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
+```
+
+Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
 
 Monitor recall by comparing results from approximate search with exact search.
 
@@ -762,46 +758,42 @@ SELECT ...
 COMMIT;
 ```
 
+## Scaling
+
+Scale pgvector the same way you scale Postgres.
+
+Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
+
+Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
+
 ## Languages
 
 Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
 
 Language | Libraries / Examples
 --- | ---
-Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada)
-Algol | [pgvector-algol](https://github.com/pgvector/pgvector-algol)
 C | [pgvector-c](https://github.com/pgvector/pgvector-c)
 C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
 C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
-COBOL | [pgvector-cobol](https://github.com/pgvector/pgvector-cobol)
 Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
-D | [pgvector-d](https://github.com/pgvector/pgvector-d)
 Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
 Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
-Erlang | [pgvector-erlang](https://github.com/pgvector/pgvector-erlang)
-Fortran | [pgvector-fortran](https://github.com/pgvector/pgvector-fortran)
-Gleam | [pgvector-gleam](https://github.com/pgvector/pgvector-gleam)
 Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
 Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
 Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
 JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
-Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl)
+Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
 Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
 Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
 Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
 OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml)
-Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal)
 Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
 PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
-Prolog | [pgvector-prolog](https://github.com/pgvector/pgvector-prolog)
 Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
 R | [pgvector-r](https://github.com/pgvector/pgvector-r)
-Racket | [pgvector-racket](https://github.com/pgvector/pgvector-racket)
-Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
 Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
 Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
 Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
-Tcl | [pgvector-tcl](https://github.com/pgvector/pgvector-tcl)
 Zig | [pgvector-zig](https://github.com/pgvector/pgvector-zig)
 
 ## Frequently Asked Questions
@@ -816,11 +808,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
 
 #### What if I want to index vectors with more than 2,000 dimensions?
 
-You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
+You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
 
 #### Can I store vectors with different dimensions in the same column?
 
-You can use `vector` as the type (instead of `vector(n)`).
+You can use `vector` as the type (instead of `vector(3)`).
 
 ```sql
 CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
@@ -875,8 +867,6 @@ No, but like other index types, you’ll likely see better performance if they d
 SELECT pg_size_pretty(pg_relation_size('index_name'));
 ```
 
-Use [half-precision indexing](#half-precision-indexing) or [binary quantization](#binary-quantization) for smaller indexes.
-
 ## Troubleshooting
 
 #### Why isn’t a query using an index?
@@ -922,7 +912,7 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
 
 #### Why are there less results for a query after adding an HNSW index?
 
-Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`), which is 40 by default. There may be even less results due to dead tuples or filtering conditions in the query. Enabling [iterative index scans](#iterative-index-scans) can help address this.
+Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
 
 Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
 
@@ -934,7 +924,7 @@ The index was likely created with too little data for the number of lists. Drop
 DROP INDEX index_name;
 ```
 
-Results can also be limited by the number of probes (`ivfflat.probes`). Enabling [iterative index scans](#iterative-index-scans) can address this.
+Results can also be limited by the number of probes (`ivfflat.probes`).
 
 Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
 
@@ -1070,7 +1060,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
 If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
 
 ```sh
-export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config
+export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
 ```
 
 Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1081,11 +1071,11 @@ sudo --preserve-env=PG_CONFIG make install
 
 A few common paths on Mac are:
 
-- EDB installer - `/Library/PostgreSQL/18/bin/pg_config`
-- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config`
-- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config`
+- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
+- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
+- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
 
-Note: Replace `18` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### Missing Header
 
@@ -1094,20 +1084,14 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
 For Ubuntu and Debian, use:
 
 ```sh
-sudo apt install postgresql-server-dev-18
+sudo apt install postgresql-server-dev-17
 ```
 
-Note: Replace `18` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### Missing SDK
 
-If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists.
-
-```sh
-pg_config --cppflags
-```
-
-Reinstall Postgres to fix this.
+If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
 
 ### Portability
 
@@ -1125,14 +1109,6 @@ make OPTFLAGS=""
 
 If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
 
-### Mismatched Architecture
-
-If compilation fails with `error C2196: case value '4' already used`, make sure you’re using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
-
-### Missing Symbol
-
-If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+.
-
 ### Permissions
 
 If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
@@ -1144,38 +1120,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
 Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
 
 ```sh
-docker pull pgvector/pgvector:pg18-trixie
+docker pull pgvector/pgvector:pg17
 ```
 
-This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way).
-
-Supported tags are:
-
-- `pg18-trixie`, `0.8.5-pg18-trixie`
-- `pg18-bookworm`, `0.8.5-pg18-bookworm`, `pg18`, `0.8.5-pg18`
-- `pg17-trixie`, `0.8.5-pg17-trixie`
-- `pg17-bookworm`, `0.8.5-pg17-bookworm`, `pg17`, `0.8.5-pg17`
-- `pg16-trixie`, `0.8.5-pg16-trixie`
-- `pg16-bookworm`, `0.8.5-pg16-bookworm`, `pg16`, `0.8.5-pg16`
-- `pg15-trixie`, `0.8.5-pg15-trixie`
-- `pg15-bookworm`, `0.8.5-pg15-bookworm`, `pg15`, `0.8.5-pg15`
-- `pg14-trixie`, `0.8.5-pg14-trixie`
-- `pg14-bookworm`, `0.8.5-pg14-bookworm`, `pg14`, `0.8.5-pg14`
-- `pg13-trixie`, `0.8.5-pg13-trixie`
-- `pg13-bookworm`, `0.8.5-pg13-bookworm`, `pg13`, `0.8.5-pg13`
+This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
 
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
+git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
-docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
-```
-
-If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
-
-```sh
-docker run --shm-size=1g ...
+docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
 ```
 
 ### Homebrew
@@ -1186,7 +1141,7 @@ With Homebrew Postgres, you can use:
 brew install pgvector
 ```
 
-Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas
+Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
 
 ### PGXN
 
@@ -1201,29 +1156,29 @@ pgxn install vector
 Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
 
 ```sh
-sudo apt install postgresql-18-pgvector
+sudo apt install postgresql-17-pgvector
 ```
 
-Note: Replace `18` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### Yum
 
 RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
 
 ```sh
-sudo yum install pgvector_18
+sudo yum install pgvector_17
 # or
-sudo dnf install pgvector_18
+sudo dnf install pgvector_17
 ```
 
-Note: Replace `18` with your Postgres server version
+Note: Replace `17` with your Postgres server version
 
 ### pkg
 
 Install the FreeBSD package with:
 
 ```sh
-pkg install postgresql17-pgvector
+pkg install postgresql15-pgvector
 ```
 
 or the port with:
@@ -1233,14 +1188,6 @@ cd /usr/ports/databases/pgvector
 make install
 ```
 
-### APK
-
-Install the Alpine package with:
-
-```sh
-apk add postgresql-pgvector
-```
-
 ### conda-forge
 
 With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
@@ -1273,6 +1220,36 @@ You can check the version in the current database with:
 SELECT extversion FROM pg_extension WHERE extname = 'vector';
 ```
 
+## Upgrade Notes
+
+### 0.6.0
+
+#### Postgres 12
+
+If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
+
+```sql
+ALTER TYPE vector SET (STORAGE = external);
+```
+
+Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
+
+#### Docker
+
+The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
+
+```sh
+docker pull pgvector/pgvector:pg16
+# or
+docker pull pgvector/pgvector:0.6.0-pg16
+```
+
+Also, if you’ve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
+
+```sh
+docker run --shm-size=1g ...
+```
+
 ## Thanks
 
 Thanks to:
@@ -1329,7 +1306,7 @@ make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
 To enable benchmarking:
 
 ```sh
-make clean && PG_CFLAGS="-DHNSW_BENCH -DIVFFLAT_BENCH" make && make install
+make clean && PG_CFLAGS="-DIVFFLAT_BENCH" make && make install
 ```
 
 To show memory usage:

sql/vector--0.8.0--0.8.1.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit

sql/vector--0.8.1--0.8.2.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.2'" to load this file. \quit

sql/vector--0.8.2--0.8.3.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.3'" to load this file. \quit

sql/vector--0.8.3--0.8.4.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.4'" to load this file. \quit

sql/vector--0.8.4--0.8.5.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.5'" to load this file. \quit

src/bitutils.c

@@ -31,12 +31,10 @@
 #define BIT_TARGET_CLONES
 #endif
 
-/* Use built-ins when possible for Postgres < 19 for inlining */
-#if PG_VERSION_NUM >= 190000
-#define popcount64(x) pg_popcount64(x)
-#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_INT_64) || SIZEOF_LONG == 8)
+/* Use built-ins when possible for inlining */
+#if defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_INT_64)
 #define popcount64(x) __builtin_popcountl(x)
-#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_LONG_INT_64) || SIZEOF_LONG_LONG == 8)
+#elif defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_LONG_INT_64)
 #define popcount64(x) __builtin_popcountll(x)
 #elif !defined(_MSC_VER)
 /* Fails to resolve with MSVC */
@@ -171,7 +169,7 @@ BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char
 #endif
 
 TARGET_XSAVE static bool
-SupportsAvx512Popcount(void)
+SupportsAvx512Popcount()
 {
 	unsigned int exx[4] = {0, 0, 0, 0};
 

src/bitvec.c

@@ -2,7 +2,6 @@
 
 #include "bitutils.h"
 #include "bitvec.h"
-#include "fmgr.h"
 #include "utils/varbit.h"
 #include "vector.h"
 

src/halfutils.c

@@ -1,7 +1,5 @@
 #include "postgres.h"
 
-#include <math.h>
-
 #include "halfutils.h"
 #include "halfvec.h"
 

src/halfvec.c

@@ -13,20 +13,12 @@
 #include "port.h"				/* for strtof() */
 #include "sparsevec.h"
 #include "utils/array.h"
+#include "utils/builtins.h"
 #include "utils/float.h"
-#include "utils/fmgrprotos.h"
 #include "utils/lsyscache.h"
-#include "utils/varbit.h"
+#include "utils/numeric.h"
 #include "vector.h"
 
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
-
-#if PG_VERSION_NUM >= 170000
-#include "parser/scansup.h"
-#endif
-
 #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
 #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
 
@@ -137,9 +129,9 @@ InitHalfVector(int dim)
 	return result;
 }
 
-#if PG_VERSION_NUM >= 170000
-#define halfvec_isspace(ch) scanner_isspace(ch)
-#else
+/*
+ * Check for whitespace, since array_isspace() is static
+ */
 static inline bool
 halfvec_isspace(char ch)
 {
@@ -152,7 +144,6 @@ halfvec_isspace(char ch)
 		return true;
 	return false;
 }
-#endif
 
 /*
  * Check state array
@@ -907,21 +898,8 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
 	half	   *ax = a->x;
 	VarBit	   *result = InitBitVector(a->dim);
 	unsigned char *rx = VARBITS(result);
-	int			i = 0;
-	int			count = (a->dim / 8) * 8;
 
-	/* Auto-vectorized on aarch64 */
-	for (; i < count; i += 8)
-	{
-		unsigned char result_byte = 0;
-
-		for (int j = 0; j < 8; j++)
-			result_byte |= (HalfToFloat4(ax[i + j]) > 0) << (7 - j);
-
-		rx[i / 8] = result_byte;
-	}
-
-	for (; i < a->dim; i++)
+	for (int i = 0; i < a->dim; i++)
 		rx[i / 8] |= (HalfToFloat4(ax[i]) > 0) << (7 - (i % 8));
 
 	PG_RETURN_VARBIT_P(result);

src/hnsw.c

@@ -1,41 +1,35 @@
 #include "postgres.h"
 
 #include <float.h>
-#include <limits.h>
 #include <math.h>
 
 #include "access/amapi.h"
-#include "access/genam.h"
 #include "access/reloptions.h"
 #include "commands/progress.h"
 #include "commands/vacuum.h"
-#include "fmgr.h"
 #include "hnsw.h"
 #include "miscadmin.h"
-#include "nodes/pg_list.h"
-#include "storage/lwlock.h"
 #include "utils/float.h"
 #include "utils/guc.h"
-#include "utils/relcache.h"
 #include "utils/selfuncs.h"
 #include "utils/spccache.h"
-#include "vector.h"
 
 #if PG_VERSION_NUM < 150000
 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
 #endif
 
-static const struct config_enum_entry hnsw_iterative_scan_options[] = {
-	{"off", HNSW_ITERATIVE_SCAN_OFF, false},
-	{"relaxed_order", HNSW_ITERATIVE_SCAN_RELAXED, false},
-	{"strict_order", HNSW_ITERATIVE_SCAN_STRICT, false},
+static const struct config_enum_entry hnsw_iterative_search_options[] = {
+	{"off", HNSW_ITERATIVE_SEARCH_OFF, false},
+	{"strict", HNSW_ITERATIVE_SEARCH_STRICT, false},
+	{"relaxed", HNSW_ITERATIVE_SEARCH_RELAXED, false},
+	/* TODO Change to strict before merging */
+	{"on", HNSW_ITERATIVE_SEARCH_RELAXED, false},
 	{NULL, 0, false}
 };
 
 int			hnsw_ef_search;
-int			hnsw_iterative_scan;
-int			hnsw_max_scan_tuples;
-double		hnsw_scan_mem_multiplier;
+int			hnsw_max_iterative_tuples;
+int			hnsw_iterative_search;
 int			hnsw_lock_tranche_id;
 static relopt_kind hnsw_relopt_kind;
 
@@ -59,20 +53,12 @@ HnswInitLockTranche(void)
 								  sizeof(int) * 1,
 								  &found);
 	if (!found)
-	{
-#if PG_VERSION_NUM >= 190000
-		tranche_ids[0] = LWLockNewTrancheId("HnswBuild");
-#else
 		tranche_ids[0] = LWLockNewTrancheId();
-#endif
-	}
 	hnsw_lock_tranche_id = tranche_ids[0];
 	LWLockRelease(AddinShmemInitLock);
 
-#if PG_VERSION_NUM < 190000
 	/* Per-backend registration of the tranche ID */
 	LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
-#endif
 }
 
 /*
@@ -94,19 +80,16 @@ HnswInit(void)
 							"Valid range is 1..1000.", &hnsw_ef_search,
 							HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
 
-	DefineCustomEnumVariable("hnsw.iterative_scan", "Sets the mode for iterative scans",
-							 NULL, &hnsw_iterative_scan,
-							 HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
+	/* TODO Change name */
+	DefineCustomEnumVariable("hnsw.streaming", "Iterative search mode",
+							 NULL, &hnsw_iterative_search,
+							 HNSW_ITERATIVE_SEARCH_OFF, hnsw_iterative_search_options, PGC_USERSET, 0, NULL, NULL, NULL);
 
-	/* This is approximate and does not affect the initial scan */
-	DefineCustomIntVariable("hnsw.max_scan_tuples", "Sets the max number of tuples to visit for iterative scans",
-							NULL, &hnsw_max_scan_tuples,
-							20000, 1, INT_MAX, PGC_USERSET, 0, NULL, NULL, NULL);
-
-	/* Same range as hash_mem_multiplier */
-	DefineCustomRealVariable("hnsw.scan_mem_multiplier", "Sets the multiple of work_mem to use for iterative scans",
-							 NULL, &hnsw_scan_mem_multiplier,
-							 1, 1, 1000, PGC_USERSET, 0, NULL, NULL, NULL);
+	/* TODO Change name */
+	/* TODO Ensure ivfflat.max_probes uses same value for "all" */
+	DefineCustomIntVariable("hnsw.ef_stream", "Sets the max number of additional candidates to visit for streaming search",
+							"-1 means all", &hnsw_max_iterative_tuples,
+							HNSW_DEFAULT_EF_STREAM, HNSW_MIN_EF_STREAM, HNSW_MAX_EF_STREAM, PGC_USERSET, 0, NULL, NULL, NULL);
 
 	MarkGUCPrefixReserved("hnsw");
 }
@@ -145,17 +128,13 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	Relation	index;
 
 	/* Never use index without order */
-	if (path->indexorderbys == NIL)
+	if (path->indexorderbys == NULL)
 	{
 		*indexStartupCost = get_float8_infinity();
 		*indexTotalCost = get_float8_infinity();
 		*indexSelectivity = 0;
 		*indexCorrelation = 0;
 		*indexPages = 0;
-#if PG_VERSION_NUM >= 180000
-		/* See "On disable_cost" thread on pgsql-hackers */
-		path->path.disabled_nodes = 2;
-#endif
 		return;
 	}
 
@@ -267,64 +246,6 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
 Datum
 hnswhandler(PG_FUNCTION_ARGS)
 {
-#if PG_VERSION_NUM >= 190000
-	static const IndexAmRoutine amroutine = {
-		.type = T_IndexAmRoutine,
-		.amstrategies = 0,
-		.amsupport = 3,
-		.amoptsprocnum = 0,
-		.amcanorder = false,
-		.amcanorderbyop = true,
-		.amcanhash = false,
-		.amconsistentequality = false,
-		.amconsistentordering = false,
-		.amcanbackward = false,
-		.amcanunique = false,
-		.amcanmulticol = false,
-		.amoptionalkey = true,
-		.amsearcharray = false,
-		.amsearchnulls = false,
-		.amstorage = false,
-		.amclusterable = false,
-		.ampredlocks = false,
-		.amcanparallel = false,
-		.amcanbuildparallel = true,
-		.amcaninclude = false,
-		.amusemaintenanceworkmem = false,
-		.amsummarizing = false,
-		.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
-		.amkeytype = InvalidOid,
-
-		.ambuild = hnswbuild,
-		.ambuildempty = hnswbuildempty,
-		.aminsert = hnswinsert,
-		.aminsertcleanup = NULL,
-		.ambulkdelete = hnswbulkdelete,
-		.amvacuumcleanup = hnswvacuumcleanup,
-		.amcanreturn = NULL,
-		.amcostestimate = hnswcostestimate,
-		.amgettreeheight = NULL,
-		.amoptions = hnswoptions,
-		.amproperty = NULL,
-		.ambuildphasename = hnswbuildphasename,
-		.amvalidate = hnswvalidate,
-		.amadjustmembers = NULL,
-		.ambeginscan = hnswbeginscan,
-		.amrescan = hnswrescan,
-		.amgettuple = hnswgettuple,
-		.amgetbitmap = NULL,
-		.amendscan = hnswendscan,
-		.ammarkpos = NULL,
-		.amrestrpos = NULL,
-		.amestimateparallelscan = NULL,
-		.aminitparallelscan = NULL,
-		.amparallelrescan = NULL,
-		.amtranslatestrategy = NULL,
-		.amtranslatecmptype = NULL,
-	};
-
-	PG_RETURN_POINTER(&amroutine);
-#else
 	IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
 
 	amroutine->amstrategies = 0;
@@ -332,11 +253,6 @@ hnswhandler(PG_FUNCTION_ARGS)
 	amroutine->amoptsprocnum = 0;
 	amroutine->amcanorder = false;
 	amroutine->amcanorderbyop = true;
-#if PG_VERSION_NUM >= 180000
-	amroutine->amcanhash = false;
-	amroutine->amconsistentequality = false;
-	amroutine->amconsistentordering = false;
-#endif
 	amroutine->amcanbackward = false;	/* can change direction mid-scan */
 	amroutine->amcanunique = false;
 	amroutine->amcanmulticol = false;
@@ -369,9 +285,6 @@ hnswhandler(PG_FUNCTION_ARGS)
 	amroutine->amvacuumcleanup = hnswvacuumcleanup;
 	amroutine->amcanreturn = NULL;
 	amroutine->amcostestimate = hnswcostestimate;
-#if PG_VERSION_NUM >= 180000
-	amroutine->amgettreeheight = NULL;
-#endif
 	amroutine->amoptions = hnswoptions;
 	amroutine->amproperty = NULL;	/* TODO AMPROP_DISTANCE_ORDERABLE */
 	amroutine->ambuildphasename = hnswbuildphasename;
@@ -392,11 +305,5 @@ hnswhandler(PG_FUNCTION_ARGS)
 	amroutine->aminitparallelscan = NULL;
 	amroutine->amparallelrescan = NULL;
 
-#if PG_VERSION_NUM >= 180000
-	amroutine->amtranslatestrategy = NULL;
-	amroutine->amtranslatecmptype = NULL;
-#endif
-
 	PG_RETURN_POINTER(amroutine);
-#endif
 }

src/hnsw.h

@@ -3,29 +3,15 @@
 
 #include "postgres.h"
 
-#include <math.h>
-
 #include "access/genam.h"
 #include "access/parallel.h"
 #include "lib/pairingheap.h"
 #include "nodes/execnodes.h"
 #include "port.h"				/* for random() */
-#include "storage/bufpage.h"
-#include "storage/condition_variable.h"
-#include "storage/lwlock.h"
-#include "storage/s_lock.h"
 #include "utils/relptr.h"
 #include "utils/sampling.h"
 #include "vector.h"
 
-#ifdef HNSW_BENCH
-#include "portability/instr_time.h"
-#endif
-
-#if PG_VERSION_NUM >= 190000
-typedef Pointer Item;
-#endif
-
 #define HNSW_MAX_DIM 2000
 #define HNSW_MAX_NNZ 1000
 
@@ -56,6 +42,9 @@ typedef Pointer Item;
 #define HNSW_DEFAULT_EF_SEARCH	40
 #define HNSW_MIN_EF_SEARCH		1
 #define HNSW_MAX_EF_SEARCH		1000
+#define HNSW_DEFAULT_EF_STREAM	-1
+#define HNSW_MIN_EF_STREAM		-1
+#define HNSW_MAX_EF_STREAM		INT_MAX
 
 /* Tuple types */
 #define HNSW_ELEMENT_TUPLE_TYPE  1
@@ -82,21 +71,6 @@ typedef Pointer Item;
 #define HnswPageGetOpaque(page)	((HnswPageOpaque) PageGetSpecialPointer(page))
 #define HnswPageGetMeta(page)	((HnswMetaPageData *) PageGetContents(page))
 
-#ifdef HNSW_BENCH
-#define HnswBench(name, code) \
-	do { \
-		instr_time	start; \
-		instr_time	duration; \
-		INSTR_TIME_SET_CURRENT(start); \
-		(code); \
-		INSTR_TIME_SET_CURRENT(duration); \
-		INSTR_TIME_SUBTRACT(duration, start); \
-		elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
-	} while (0)
-#else
-#define HnswBench(name, code) (code)
-#endif
-
 #if PG_VERSION_NUM >= 150000
 #define RandomDouble() pg_prng_double(&pg_global_prng_state)
 #define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -138,24 +112,23 @@ typedef Pointer Item;
 
 /* Variables */
 extern int	hnsw_ef_search;
-extern int	hnsw_iterative_scan;
-extern int	hnsw_max_scan_tuples;
-extern double hnsw_scan_mem_multiplier;
+extern int	hnsw_max_iterative_tuples;
+extern int	hnsw_iterative_search;
 extern int	hnsw_lock_tranche_id;
 
-typedef enum HnswIterativeScanMode
+typedef enum HnswIterativeSearchType
 {
-	HNSW_ITERATIVE_SCAN_OFF,
-	HNSW_ITERATIVE_SCAN_RELAXED,
-	HNSW_ITERATIVE_SCAN_STRICT
-}			HnswIterativeScanMode;
+	HNSW_ITERATIVE_SEARCH_OFF,
+	HNSW_ITERATIVE_SEARCH_STRICT,
+	HNSW_ITERATIVE_SEARCH_RELAXED
+}			HnswIterativeSearchType;
 
 typedef struct HnswElementData HnswElementData;
 typedef struct HnswNeighborArray HnswNeighborArray;
 
 #define HnswPtrDeclare(type, relptrtype, ptrtype) \
 	relptr_declare(type, relptrtype); \
-	typedef union { type *ptr; relptrtype relptr; } ptrtype
+	typedef union { type *ptr; relptrtype relptr; } ptrtype;
 
 /* Pointers that can be absolute or relative */
 /* Use char for DatumPtr so works with Pointer */
@@ -402,7 +375,6 @@ typedef struct HnswScanOpaqueData
 	int			m;
 	int64		tuples;
 	double		previousDistance;
-	Size		maxMemory;
 	MemoryContext tmpCtx;
 
 	/* Support functions */
@@ -427,11 +399,10 @@ typedef struct HnswVacuumState
 	HnswSupport support;
 
 	/* Variables */
-	struct tidhash_hash *deleting;
+	struct tidhash_hash *deleted;
 	BufferAccessStrategy bas;
 	HnswNeighborTuple ntup;
 	HnswElementData highestPoint;
-	HnswElementData fallbackPoint;
 
 	/* Memory */
 	MemoryContext tmpCtx;
@@ -454,7 +425,7 @@ void	   *HnswAlloc(HnswAllocator * allocator, Size size);
 HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
 HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
 void		HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
-HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec);
+HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec);
 void		HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
 void		HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
 void		HnswAddHeapTid(HnswElement element, ItemPointer heaptid);

src/hnswbuild.c

@@ -36,14 +36,11 @@
  */
 #include "postgres.h"
 
-#include <limits.h>
+#include <math.h>
 
-#include "access/genam.h"
 #include "access/parallel.h"
-#include "access/relscan.h"
 #include "access/table.h"
 #include "access/tableam.h"
-#include "access/tupdesc.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
 #include "catalog/index.h"
@@ -51,19 +48,11 @@
 #include "commands/progress.h"
 #include "hnsw.h"
 #include "miscadmin.h"
-#include "nodes/execnodes.h"
 #include "optimizer/optimizer.h"
 #include "storage/bufmgr.h"
-#include "storage/condition_variable.h"
 #include "tcop/tcopprot.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
-#include "utils/rel.h"
-#include "utils/snapmgr.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
 
 #if PG_VERSION_NUM >= 140000
 #include "utils/backend_progress.h"
@@ -80,8 +69,6 @@
 #define PARALLEL_KEY_HNSW_AREA			UINT64CONST(0xA000000000000002)
 #define PARALLEL_KEY_QUERY_TEXT			UINT64CONST(0xA000000000000003)
 
-#define HNSW_MAX_GRAPH_MEMORY (SIZE_MAX / 2)
-
 /*
  * Create the metapage
  */
@@ -411,7 +398,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
  * Update graph in memory
  */
 static void
-UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate)
+UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
 {
 	HnswGraph  *graph = buildstate->graph;
 	char	   *base = buildstate->hnswarea;
@@ -473,7 +460,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
 	HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
 
 	/* Update graph in memory */
-	UpdateGraphInMemory(support, element, m, entryPoint, buildstate);
+	UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate);
 
 	/* Release entry lock */
 	LWLockRelease(entryLock);
@@ -494,7 +481,6 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	LWLock	   *flushLock = &graph->flushLock;
 	char	   *base = buildstate->hnswarea;
 	Datum		value;
-	Size		memoryMargin;
 
 	/* Form index value */
 	if (!HnswFormIndexValue(&value, values, isnull, buildstate->typeInfo, support))
@@ -503,9 +489,6 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	/* Get datum size */
 	valueSize = VARSIZE_ANY(DatumGetPointer(value));
 
-	/* In a parallel build, add a margin so allocations never fail */
-	memoryMargin = base == NULL ? 0 : 1024 * 1024;
-
 	/* Ensure graph not flushed when inserting */
 	LWLockAcquire(flushLock, LW_SHARED);
 
@@ -527,7 +510,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 	 * Check that we have enough memory available for the new element now that
 	 * we have the allocator lock, and flush pages if needed.
 	 */
-	if (graph->memoryUsed + memoryMargin >= graph->memoryTotal)
+	if (graph->memoryUsed >= graph->memoryTotal)
 	{
 		LWLockRelease(&graph->allocatorLock);
 
@@ -562,7 +545,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
 
 	/* Copy the datum */
 	memcpy(valuePtr, DatumGetPointer(value), valueSize);
-	HnswPtrStore(base, element->value, (char *) valuePtr);
+	HnswPtrStore(base, element->value, valuePtr);
 
 	/* Create a lock for the element */
 	LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -620,7 +603,7 @@ InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
 	HnswPtrStore(base, graph->head, (HnswElement) NULL);
 	HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
 	graph->memoryUsed = 0;
-	graph->memoryTotal = Min(memoryTotal, HNSW_MAX_GRAPH_MEMORY);
+	graph->memoryTotal = memoryTotal;
 	graph->flushed = false;
 	graph->indtuples = 0;
 	SpinLockInit(&graph->lock);
@@ -661,17 +644,9 @@ static void *
 HnswSharedMemoryAlloc(Size size, void *state)
 {
 	HnswBuildState *buildstate = (HnswBuildState *) state;
-	Size		alignedSize = MAXALIGN(size);
-	void	   *chunk;
+	void	   *chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
 
-	if (alignedSize > 1024 * 1024)
-		elog(ERROR, "hnsw allocation too large");
-
-	if (buildstate->graph->memoryUsed + alignedSize > buildstate->graph->memoryTotal)
-		elog(ERROR, "hnsw allocator out of memory");
-
-	chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
-	buildstate->graph->memoryUsed += alignedSize;
+	buildstate->graph->memoryUsed += MAXALIGN(size);
 	return chunk;
 }
 
@@ -719,7 +694,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
 	/* Get support functions */
 	HnswInitSupport(&buildstate->support, index);
 
-	InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * (Size) 1024);
+	InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L);
 	buildstate->graph = &buildstate->graphData;
 	buildstate->ml = HnswGetMl(buildstate->m);
 	buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
@@ -803,11 +778,7 @@ HnswParallelScanAndInsert(Relation heapRel, Relation indexRel, HnswShared * hnsw
 	buildstate.hnswarea = hnswarea;
 	InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate);
 	scan = table_beginscan_parallel(heapRel,
-									ParallelTableScanFromHnswShared(hnswshared)
-#if PG_VERSION_NUM >= 190000
-									,SO_NONE
-#endif
-		);
+									ParallelTableScanFromHnswShared(hnswshared));
 	reltuples = table_index_build_scan(heapRel, indexRel, indexInfo,
 									   true, progress, BuildCallback,
 									   (void *) &buildstate, scan);
@@ -956,13 +927,11 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
 	/* Leave space for other objects in shared memory */
 	/* Docker has a default limit of 64 MB for shm_size */
 	/* which happens to be the default value of maintenance_work_mem */
-	esthnswarea = maintenance_work_mem * (Size) 1024;
+	esthnswarea = maintenance_work_mem * 1024L;
 	estother = 3 * 1024 * 1024;
 	if (esthnswarea > estother)
 		esthnswarea -= estother;
 
-	esthnswarea = Min(esthnswarea, HNSW_MAX_GRAPH_MEMORY);
-
 	shm_toc_estimate_chunk(&pcxt->estimator, esthnswarea);
 	shm_toc_estimate_keys(&pcxt->estimator, 2);
 
@@ -1005,7 +974,8 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
 								  snapshot);
 
 	hnswarea = (char *) shm_toc_allocate(pcxt->toc, esthnswarea);
-	InitGraph(&hnswshared->graphData, hnswarea, esthnswarea);
+	/* Report less than allocated so never fails */
+	InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
 
 	/*
 	 * Avoid base address for relptr for Postgres < 14.5
@@ -1084,7 +1054,7 @@ ComputeParallelWorkers(Relation heap, Relation index)
  * Build graph
  */
 static void
-BuildGraph(HnswBuildState * buildstate)
+BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
 {
 	int			parallel_workers = 0;
 
@@ -1132,7 +1102,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
 
 	InitBuildState(buildstate, heap, index, indexInfo, forkNum);
 
-	BuildGraph(buildstate);
+	BuildGraph(buildstate, forkNum);
 
 	if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
 		log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);

src/hnswinsert.c

@@ -1,19 +1,13 @@
 #include "postgres.h"
 
-#include "access/genam.h"
+#include <math.h>
+
 #include "access/generic_xlog.h"
 #include "hnsw.h"
-#include "nodes/execnodes.h"
 #include "storage/bufmgr.h"
 #include "storage/lmgr.h"
-#include "storage/lwlock.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
-#include "utils/rel.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
 
 /*
  * Get the insert page
@@ -666,7 +660,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
  * Update graph on disk
  */
 static void
-UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building)
+UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
 {
 	BlockNumber newInsertPage = InvalidBlockNumber;
 
@@ -714,7 +708,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
 
 	/* Create an element */
 	element = HnswInitElement(base, heaptid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
-	HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
+	HnswPtrStore(base, element->value, DatumGetPointer(value));
 
 	/* Prevent concurrent inserts when likely updating entry point */
 	if (entryPoint == NULL || element->level > entryPoint->level)
@@ -734,7 +728,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
 	HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
 
 	/* Update graph on disk */
-	UpdateGraphOnDisk(index, support, element, m, entryPoint, building);
+	UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building);
 
 	/* Release lock */
 	UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);

src/hnswscan.c

@@ -1,23 +1,12 @@
 #include "postgres.h"
 
-#include <limits.h>
-
-#include "access/genam.h"
 #include "access/relscan.h"
 #include "hnsw.h"
-#include "lib/pairingheap.h"
-#include "miscadmin.h"
-#include "nodes/pg_list.h"
 #include "pgstat.h"
+#include "storage/bufmgr.h"
 #include "storage/lmgr.h"
 #include "utils/float.h"
 #include "utils/memutils.h"
-#include "utils/relcache.h"
-#include "utils/snapmgr.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
 
 /*
  * Algorithm 5 from paper
@@ -52,7 +41,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
 		ep = w;
 	}
 
-	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples);
+	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_search != HNSW_ITERATIVE_SEARCH_OFF ? &so->discarded : NULL, true, &so->tuples);
 }
 
 /*
@@ -113,17 +102,6 @@ GetScanValue(IndexScanDesc scan)
 	return value;
 }
 
-#if defined(HNSW_MEMORY)
-/*
- * Show memory usage
- */
-static void
-ShowMemoryUsage(HnswScanOpaque so)
-{
-	elog(INFO, "memory: %zu KB, tuples: " INT64_FORMAT, MemoryContextMemAllocated(so->tmpCtx, false) / 1024, so->tuples);
-}
-#endif
-
 /*
  * Prepare for an index scan
  */
@@ -132,29 +110,21 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
 {
 	IndexScanDesc scan;
 	HnswScanOpaque so;
-	double		maxMemory;
 
 	scan = RelationGetIndexScan(index, nkeys, norderbys);
 
 	so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
 	so->typeInfo = HnswGetTypeInfo(index);
+	so->first = true;
+	so->v.tids = NULL;
+	so->discarded = NULL;
+	so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
+									   "Hnsw scan temporary context",
+									   ALLOCSET_DEFAULT_SIZES);
 
 	/* Set support functions */
 	HnswInitSupport(&so->support, index);
 
-	/*
-	 * Use a lower max allocation size than default to allow scanning more
-	 * tuples for iterative search before exceeding work_mem
-	 */
-	so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
-									   "Hnsw scan temporary context",
-									   0, 8 * 1024, 256 * 1024);
-
-	/* Calculate max memory */
-	/* Add 256 extra bytes to fill last block when close */
-	maxMemory = (double) work_mem * hnsw_scan_mem_multiplier * 1024.0 + 256;
-	so->maxMemory = Min(maxMemory, (double) (SIZE_MAX / 2));
-
 	scan->opaque = so;
 
 	return scan;
@@ -168,10 +138,13 @@ hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int no
 {
 	HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
 
+	if (so->v.tids != NULL)
+		tidhash_reset(so->v.tids);
+
+	if (so->discarded != NULL)
+		pairingheap_reset(so->discarded);
+
 	so->first = true;
-	/* v and discarded are allocated in tmpCtx */
-	so->v.tids = NULL;
-	so->discarded = NULL;
 	so->tuples = 0;
 	so->previousDistance = -get_float8_infinity();
 	MemoryContextReset(so->tmpCtx);
@@ -204,10 +177,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 
 		/* Count index scan for stats */
 		pgstat_count_index_scan(scan->indexRelation);
-#if PG_VERSION_NUM >= 180000
-		if (scan->instrument)
-			scan->instrument->nsearches++;
-#endif
 
 		/* Safety check */
 		if (scan->orderByData == NULL)
@@ -235,7 +204,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 		so->first = false;
 
 #if defined(HNSW_MEMORY)
-		ShowMemoryUsage(so);
+		elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
 #endif
 	}
 
@@ -248,15 +217,15 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 
 		if (list_length(so->w) == 0)
 		{
-			if (hnsw_iterative_scan == HNSW_ITERATIVE_SCAN_OFF)
+			if (hnsw_iterative_search == HNSW_ITERATIVE_SEARCH_OFF)
 				break;
 
 			/* Empty index */
 			if (so->discarded == NULL)
 				break;
 
-			/* Reached max number of tuples or memory limit */
-			if (so->tuples >= hnsw_max_scan_tuples || MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
+			/* Reached max number of additional tuples */
+			if (hnsw_max_iterative_tuples != -1 && so->tuples >= hnsw_ef_search + hnsw_max_iterative_tuples)
 			{
 				if (pairingheap_is_empty(so->discarded))
 					break;
@@ -264,6 +233,21 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 				/* Return remaining tuples */
 				so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
 			}
+			/* Prevent scans from consuming too much memory */
+			else if (MemoryContextMemAllocated(so->tmpCtx, false) > (Size) work_mem * 1024L)
+			{
+				if (pairingheap_is_empty(so->discarded))
+				{
+					ereport(DEBUG1,
+							(errmsg("hnsw index scan exceeded work_mem after " INT64_FORMAT " tuples", so->tuples),
+							 errhint("Increase work_mem to scan more tuples.")));
+
+					break;
+				}
+
+				/* Return remaining tuples */
+				so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
+			}
 			else
 			{
 				/*
@@ -282,7 +266,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 				UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
 
 #if defined(HNSW_MEMORY)
-				ShowMemoryUsage(so);
+				elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
 #endif
 			}
 
@@ -299,7 +283,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 			so->w = list_delete_last(so->w);
 
 			/* Mark memory as free for next iteration */
-			if (hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF)
+			if (hnsw_iterative_search != HNSW_ITERATIVE_SEARCH_OFF)
 			{
 				pfree(element);
 				pfree(sc);
@@ -310,7 +294,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 
 		heaptid = &element->heaptids[--element->heaptidsLength];
 
-		if (hnsw_iterative_scan == HNSW_ITERATIVE_SCAN_STRICT)
+		if (hnsw_iterative_search == HNSW_ITERATIVE_SEARCH_STRICT)
 		{
 			if (sc->distance < so->previousDistance)
 				continue;

src/hnswutils.c

@@ -2,24 +2,18 @@
 
 #include <math.h>
 
-#include "access/genam.h"
 #include "access/generic_xlog.h"
+#include "catalog/pg_type.h"
+#include "catalog/pg_type_d.h"
 #include "common/hashfn.h"
 #include "fmgr.h"
 #include "hnsw.h"
 #include "lib/pairingheap.h"
-#include "nodes/pg_list.h"
-#include "port/atomics.h"
 #include "sparsevec.h"
 #include "storage/bufmgr.h"
 #include "utils/datum.h"
 #include "utils/memdebug.h"
 #include "utils/rel.h"
-#include "vector.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
 
 #if PG_VERSION_NUM < 170000
 static inline uint64
@@ -262,7 +256,7 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
 
 	HnswInitNeighbors(base, element, m, allocator);
 
-	HnswPtrStore(base, element->value, (char *) NULL);
+	HnswPtrStore(base, element->value, (Pointer) NULL);
 
 	return element;
 }
@@ -288,7 +282,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
 	element->blkno = blkno;
 	element->offno = offno;
 	HnswPtrStore(base, element->neighbors, (HnswNeighborArrayPtr *) NULL);
-	HnswPtrStore(base, element->value, (char *) NULL);
+	HnswPtrStore(base, element->value, (Pointer) NULL);
 	return element;
 }
 
@@ -514,7 +508,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
 		char	   *base = NULL;
 		Datum		value = datumCopy(PointerGetDatum(&etup->data), false, -1);
 
-		HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
+		HnswPtrStore(base, element->value, DatumGetPointer(value));
 	}
 }
 
@@ -546,9 +540,6 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, Hns
 
 	Assert(HnswIsElementTuple(etup));
 
-	if (unlikely(etup->deleted))
-		elog(ERROR, "cannot load deleted element");
-
 	/* Calculate distance */
 	if (distance != NULL)
 	{
@@ -590,34 +581,21 @@ GetElementDistance(char *base, HnswElement element, HnswQuery * q, HnswSupport *
 	return HnswGetDistance(q->value, value, support);
 }
 
-/*
- * Allocate a search candidate
- */
-static HnswSearchCandidate *
-HnswInitSearchCandidate(char *base, HnswElement element, double distance)
-{
-	HnswSearchCandidate *sc = palloc(sizeof(HnswSearchCandidate));
-
-	HnswPtrStore(base, sc->element, element);
-	sc->distance = distance;
-	return sc;
-}
-
 /*
  * Create a candidate for the entry point
  */
 HnswSearchCandidate *
 HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec)
 {
+	HnswSearchCandidate *sc = palloc(sizeof(HnswSearchCandidate));
 	bool		inMemory = index == NULL;
-	double		distance;
 
+	HnswPtrStore(base, sc->element, entryPoint);
 	if (inMemory)
-		distance = GetElementDistance(base, entryPoint, q, support);
+		sc->distance = GetElementDistance(base, entryPoint, q, support);
 	else
-		HnswLoadElement(entryPoint, &distance, q, index, support, loadVec, NULL);
-
-	return HnswInitSearchCandidate(base, entryPoint, distance);
+		HnswLoadElement(entryPoint, &sc->distance, q, index, support, loadVec, NULL);
+	return sc;
 }
 
 /*
@@ -866,7 +844,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
 		{
 			AddToVisited(base, v, sc->element, inMemory, &found);
 
-			/* OK to count elements instead of tuples */
 			if (tuples != NULL)
 				(*tuples)++;
 		}
@@ -899,7 +876,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
 		else
 			HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
 
-		/* OK to count elements instead of tuples */
 		if (tuples != NULL)
 			(*tuples) += unvisitedLength;
 
@@ -931,12 +907,14 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
 					continue;
 			}
 
-			if (!(eDistance < f->distance || alwaysAdd))
+			if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
 			{
 				if (discarded != NULL)
 				{
 					/* Create a new candidate */
-					e = HnswInitSearchCandidate(base, eElement, eDistance);
+					e = palloc(sizeof(HnswSearchCandidate));
+					HnswPtrStore(base, e->element, eElement);
+					e->distance = eDistance;
 					pairingheap_add(*discarded, &e->w_node);
 				}
 
@@ -948,7 +926,9 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
 				continue;
 
 			/* Create a new candidate */
-			e = HnswInitSearchCandidate(base, eElement, eDistance);
+			e = palloc(sizeof(HnswSearchCandidate));
+			HnswPtrStore(base, e->element, eElement);
+			e->distance = eDistance;
 			pairingheap_add(C, &e->c_node);
 			pairingheap_add(W, &e->w_node);
 
@@ -1402,7 +1382,7 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
-}
+};
 
 FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
 Datum
@@ -1415,7 +1395,7 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
-}
+};
 
 FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
 Datum
@@ -1428,4 +1408,4 @@ hnsw_sparsevec_support(PG_FUNCTION_ARGS)
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
-}
+};

src/hnswvacuum.c

@@ -1,30 +1,21 @@
 #include "postgres.h"
 
-#include "access/genam.h"
+#include <math.h>
+
 #include "access/generic_xlog.h"
 #include "commands/vacuum.h"
 #include "hnsw.h"
-#include "nodes/pg_list.h"
 #include "storage/bufmgr.h"
 #include "storage/lmgr.h"
 #include "utils/memutils.h"
-#include "utils/rel.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
-
-#if PG_VERSION_NUM >= 180000
-#define vacuum_delay_point() vacuum_delay_point(false)
-#endif
 
 /*
- * Check if deletion list contains an element
+ * Check if deleted list contains an index TID
  */
 static bool
-DeletingElement(tidhash_hash * deleting, ItemPointer indextid)
+DeletedContains(tidhash_hash * deleted, ItemPointer indextid)
 {
-	return tidhash_lookup(deleting, *indextid) != NULL;
+	return tidhash_lookup(deleted, *indextid) != NULL;
 }
 
 /*
@@ -37,20 +28,17 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 {
 	BlockNumber blkno = HNSW_HEAD_BLKNO;
 	HnswElement highestPoint = &vacuumstate->highestPoint;
-	HnswElement fallbackPoint = &vacuumstate->fallbackPoint;
 	Relation	index = vacuumstate->index;
 	BufferAccessStrategy bas = vacuumstate->bas;
+	HnswElement entryPoint = HnswGetEntryPoint(vacuumstate->index);
 	IndexBulkDeleteResult *stats = vacuumstate->stats;
 
-	/* Store separately since HnswElement level is uint8 */
+	/* Store separately since highestPoint.level is uint8 */
 	int			highestLevel = -1;
-	int			fallbackLevel = -1;
 
-	/* Initialize highest point and fallback point */
+	/* Initialize highest point */
 	highestPoint->blkno = InvalidBlockNumber;
 	highestPoint->offno = InvalidOffsetNumber;
-	fallbackPoint->blkno = InvalidBlockNumber;
-	fallbackPoint->offno = InvalidOffsetNumber;
 
 	while (BlockNumberIsValid(blkno))
 	{
@@ -80,14 +68,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 			if (!HnswIsElementTuple(etup))
 				continue;
 
-			/*
-			 * Skip deleted tuples. It is important they are not added to the
-			 * deletion list to avoid false positives in NeedsUpdated and
-			 * ConfirmRepaired.
-			 */
-			if (etup->deleted)
-				continue;
-
 			if (ItemPointerIsValid(&etup->heaptids[0]))
 			{
 				for (int i = 0; i < HNSW_HEAPTIDS; i++)
@@ -121,40 +101,23 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 			if (!ItemPointerIsValid(&etup->heaptids[0]))
 			{
-				ItemPointerData indextid;
+				ItemPointerData ip;
 				bool		found;
 
-				/* Add to deletion list */
-				ItemPointerSet(&indextid, blkno, offno);
+				/* Add to deleted list */
+				ItemPointerSet(&ip, blkno, offno);
 
-				tidhash_insert(vacuumstate->deleting, indextid, &found);
+				tidhash_insert(vacuumstate->deleted, ip, &found);
 				Assert(!found);
 			}
-			else if (etup->level > highestLevel)
+			else if (etup->level > highestLevel && !(entryPoint != NULL && blkno == entryPoint->blkno && offno == entryPoint->offno))
 			{
-				if (BlockNumberIsValid(highestPoint->blkno))
-				{
-					/* Current highest point becomes fallback */
-					fallbackPoint->blkno = highestPoint->blkno;
-					fallbackPoint->offno = highestPoint->offno;
-					fallbackPoint->level = highestPoint->level;
-					fallbackLevel = highestLevel;
-				}
-
-				/* Keep track of highest point */
+				/* Keep track of highest non-entry point */
 				highestPoint->blkno = blkno;
 				highestPoint->offno = offno;
 				highestPoint->level = etup->level;
 				highestLevel = etup->level;
 			}
-			else if (etup->level > fallbackLevel)
-			{
-				/* Keep track of second highest point */
-				fallbackPoint->blkno = blkno;
-				fallbackPoint->offno = offno;
-				fallbackPoint->level = etup->level;
-				fallbackLevel = etup->level;
-			}
 		}
 
 		blkno = HnswPageGetOpaque(page)->nextblkno;
@@ -166,10 +129,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
 
 		UnlockReleaseBuffer(buf);
 	}
-
-#ifdef HNSW_MEMORY
-	elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(CurrentMemoryContext, true) / 1024);
-#endif
 }
 
 /*
@@ -200,8 +159,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
 		if (!ItemPointerIsValid(indextid))
 			continue;
 
-		/* Check if in deletion list */
-		if (DeletingElement(vacuumstate->deleting, indextid))
+		/* Check if in deleted list */
+		if (DeletedContains(vacuumstate->deleted, indextid))
 		{
 			needsUpdated = true;
 			break;
@@ -210,8 +169,7 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
 
 	/* Also update if layer 0 is not full */
 	/* This could indicate too many candidates being deleted during insert */
-	/* There should always be more than zero indextids, but check for safety */
-	if (!needsUpdated && ntup->count > 0)
+	if (!needsUpdated)
 		needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
 
 	UnlockReleaseBuffer(buf);
@@ -297,27 +255,12 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
 		/* Get a shared lock */
 		LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
 
-		/* Get latest entry point */
-		entryPoint = HnswGetEntryPoint(index);
+		/* Load element */
+		HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
 
-		/* Use fallback point if highest point is entry point */
-		if (entryPoint != NULL && entryPoint->blkno == highestPoint->blkno && entryPoint->offno == highestPoint->offno)
-		{
-			highestPoint = &vacuumstate->fallbackPoint;
-
-			if (!BlockNumberIsValid(highestPoint->blkno))
-				highestPoint = NULL;
-		}
-
-		if (highestPoint != NULL)
-		{
-			/* Load element */
-			HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
-
-			/* Repair if needed */
-			if (NeedsUpdated(vacuumstate, highestPoint))
-				RepairGraphElement(vacuumstate, highestPoint, entryPoint);
-		}
+		/* Repair if needed */
+		if (NeedsUpdated(vacuumstate, highestPoint))
+			RepairGraphElement(vacuumstate, highestPoint, HnswGetEntryPoint(index));
 
 		/* Release lock */
 		UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock);
@@ -335,7 +278,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
 
 		ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno);
 
-		if (DeletingElement(vacuumstate->deleting, &epData))
+		if (DeletedContains(vacuumstate->deleted, &epData))
 		{
 			/*
 			 * Replace the entry point with the highest point. If highest
@@ -421,10 +364,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
 			if (!HnswIsElementTuple(etup))
 				continue;
 
-			/* Skip deleted tuples */
-			if (etup->deleted)
-				continue;
-
 			/* Skip updating neighbors if being deleted */
 			if (!ItemPointerIsValid(&etup->heaptids[0]))
 				continue;
@@ -488,103 +427,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
 		/* Reset memory context */
 		MemoryContextSwitchTo(oldCtx);
 		MemoryContextReset(vacuumstate->tmpCtx);
-
-#ifdef HNSW_VACUUM_PROGRESS
-		if (!BlockNumberIsValid(blkno) || (blkno - HNSW_HEAD_BLKNO) % 1000 == 0)
-		{
-			BlockNumber totalBlocks = RelationGetNumberOfBlocks(index);
-			BlockNumber currentBlocks = BlockNumberIsValid(blkno) ? blkno : totalBlocks;
-
-			elog(INFO, "hnsw vacuum progress: %.1f%%", 100.0 * currentBlocks / totalBlocks);
-		}
-#endif
-	}
-}
-
-/*
- * Confirm graph was repaired
- */
-static void
-ConfirmRepaired(HnswVacuumState * vacuumstate)
-{
-	BlockNumber blkno = HNSW_HEAD_BLKNO;
-	Relation	index = vacuumstate->index;
-	BufferAccessStrategy bas = vacuumstate->bas;
-
-	while (BlockNumberIsValid(blkno))
-	{
-		Buffer		buf;
-		Page		page;
-		OffsetNumber offno;
-		OffsetNumber maxoffno;
-
-		vacuum_delay_point();
-
-		buf = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bas);
-		LockBuffer(buf, BUFFER_LOCK_SHARE);
-		page = BufferGetPage(buf);
-		maxoffno = PageGetMaxOffsetNumber(page);
-
-		/* Iterate over nodes */
-		for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
-		{
-			HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
-			HnswNeighborTuple ntup;
-			Buffer		nbuf;
-			Page		npage;
-			BlockNumber neighborPage;
-			OffsetNumber neighborOffno;
-
-			/* Skip neighbor tuples */
-			if (!HnswIsElementTuple(etup))
-				continue;
-
-			/* Skip deleted tuples */
-			if (etup->deleted)
-				continue;
-
-			/* Skip if being deleted */
-			if (!ItemPointerIsValid(&etup->heaptids[0]))
-				continue;
-
-			/* Get neighbor page */
-			neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
-			neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
-
-			if (neighborPage == blkno)
-			{
-				nbuf = buf;
-				npage = page;
-			}
-			else
-			{
-				nbuf = ReadBufferExtended(index, MAIN_FORKNUM, neighborPage, RBM_NORMAL, bas);
-				LockBuffer(nbuf, BUFFER_LOCK_SHARE);
-				npage = BufferGetPage(nbuf);
-			}
-
-			ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
-
-			/* Check neighbors */
-			for (int i = 0; i < ntup->count; i++)
-			{
-				ItemPointer indextid = &ntup->indextids[i];
-
-				if (!ItemPointerIsValid(indextid))
-					continue;
-
-				/* Check if in deletion list */
-				if (DeletingElement(vacuumstate->deleting, indextid))
-					elog(ERROR, "hnsw graph not repaired");
-			}
-
-			if (nbuf != buf)
-				UnlockReleaseBuffer(nbuf);
-		}
-
-		blkno = HnswPageGetOpaque(page)->nextblkno;
-
-		UnlockReleaseBuffer(buf);
 	}
 }
 
@@ -600,15 +442,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 	BufferAccessStrategy bas = vacuumstate->bas;
 
 	/*
-	 * Wait for inserts and index scans to complete. Inserts and scans before
-	 * this point may visit tuples about to be deleted. Inserts and scans
-	 * after this point will not, since the graph has been repaired.
+	 * Wait for index scans to complete. Scans before this point may contain
+	 * tuples about to be deleted. Scans after this point will not, since the
+	 * graph has been repaired.
 	 */
-	LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
-	UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
-
-	ConfirmRepaired(vacuumstate);
-
 	LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
 	UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
 
@@ -683,9 +520,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 			ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
 
 			/* Overwrite element */
-			/* Use memset instead of MemSet to keep clang-tidy happy */
 			etup->deleted = 1;
-			memset(&etup->data, 0, VARSIZE_ANY(&etup->data));
+			MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
 
 			/* Overwrite neighbors */
 			for (int i = 0; i < ntup->count; i++)
@@ -756,7 +592,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 	HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
 
 	/* Create hash table */
-	vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
+	vacuumstate->deleted = tidhash_create(CurrentMemoryContext, 256, NULL);
 }
 
 /*
@@ -765,7 +601,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 static void
 FreeVacuumState(HnswVacuumState * vacuumstate)
 {
-	tidhash_destroy(vacuumstate->deleting);
+	tidhash_destroy(vacuumstate->deleted);
 	FreeAccessStrategy(vacuumstate->bas);
 	pfree(vacuumstate->ntup);
 	MemoryContextDelete(vacuumstate->tmpCtx);
@@ -783,13 +619,13 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 	InitVacuumState(&vacuumstate, info, stats, callback, callback_state);
 
 	/* Pass 1: Remove heap TIDs */
-	HnswBench("RemoveHeapTids", RemoveHeapTids(&vacuumstate));
+	RemoveHeapTids(&vacuumstate);
 
 	/* Pass 2: Repair graph */
-	HnswBench("RepairGraph", RepairGraph(&vacuumstate));
+	RepairGraph(&vacuumstate);
 
-	/* Passes 3 and 4: Confirm repaired and mark as deleted */
-	HnswBench("MarkDeleted", MarkDeleted(&vacuumstate));
+	/* Pass 3: Mark as deleted */
+	MarkDeleted(&vacuumstate);
 
 	FreeVacuumState(&vacuumstate);
 

src/ivfbuild.c

@@ -2,37 +2,23 @@
 
 #include <float.h>
 
-#include "access/genam.h"
-#include "access/generic_xlog.h"
-#include "access/itup.h"
-#include "access/relscan.h"
 #include "access/table.h"
 #include "access/tableam.h"
-#include "access/tupdesc.h"
 #include "access/parallel.h"
 #include "access/xact.h"
-#include "access/xloginsert.h"
+#include "bitvec.h"
 #include "catalog/index.h"
 #include "catalog/pg_operator_d.h"
 #include "catalog/pg_type_d.h"
 #include "commands/progress.h"
-#include "fmgr.h"
+#include "halfvec.h"
 #include "ivfflat.h"
 #include "miscadmin.h"
-#include "nodes/execnodes.h"
 #include "optimizer/optimizer.h"
 #include "storage/bufmgr.h"
-#include "storage/condition_variable.h"
 #include "tcop/tcopprot.h"
 #include "utils/memutils.h"
-#include "utils/rel.h"
-#include "utils/sampling.h"
-#include "utils/snapmgr.h"
-#include "utils/tuplesort.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
+#include "vector.h"
 
 #if PG_VERSION_NUM >= 140000
 #include "utils/backend_progress.h"
@@ -63,13 +49,15 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
 	Datum		value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
 
 	/*
-	 * Check with KMEANS_NORM_PROC that the value can be normalized since
-	 * spherical distance function expects unit vectors
+	 * Normalize with KMEANS_NORM_PROC since spherical distance function
+	 * expects unit vectors
 	 */
 	if (buildstate->kmeansnormprocinfo != NULL)
 	{
 		if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
 			return;
+
+		value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
 	}
 
 	if (samples->length < targsamples)
@@ -80,7 +68,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
 	else
 	{
 		if (buildstate->rowstoskip < 0)
-			buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, buildstate->samplerows, targsamples);
+			buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
 
 		if (buildstate->rowstoskip <= 0)
 		{
@@ -96,9 +84,6 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
 
 		buildstate->rowstoskip -= 1;
 	}
-
-	/* Increment after reservoir_get_next_S */
-	buildstate->samplerows += 1;
 }
 
 /*
@@ -135,7 +120,6 @@ SampleRows(IvfflatBuildState * buildstate)
 	int			targsamples = buildstate->samples->maxlen;
 	BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
 
-	buildstate->samplerows = 0;
 	buildstate->rowstoskip = -1;
 
 	BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
@@ -145,21 +129,16 @@ SampleRows(IvfflatBuildState * buildstate)
 	{
 		BlockNumber targblock = BlockSampler_Next(&buildstate->bs);
 
-		/* Set anyvisible to false like table_index_build_scan */
 		table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
-									 false, false, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
+									 false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
 	}
-
-	/* Normalize if needed */
-	if (buildstate->kmeansnormprocinfo != NULL)
-		IvfflatNormVectors(buildstate->typeInfo, buildstate->collation, buildstate->samples, buildstate->tmpCtx);
 }
 
 /*
  * Add tuple to sort
  */
 static void
-AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
+AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
 {
 	double		distance;
 	double		minDistance = DBL_MAX;
@@ -236,7 +215,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
 	oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
 
 	/* Add tuple to sort */
-	AddTupleToSort(tid, values, buildstate);
+	AddTupleToSort(index, tid, values, buildstate);
 
 	/* Reset memory context */
 	MemoryContextSwitchTo(oldCtx);
@@ -249,11 +228,11 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
 static inline void
 GetNextTuple(Tuplesortstate *sortstate, TupleDesc tupdesc, TupleTableSlot *slot, IndexTuple *itup, int *list)
 {
+	Datum		value;
+	bool		isnull;
+
 	if (tuplesort_gettupleslot(sortstate, true, false, slot, NULL))
 	{
-		Datum		value;
-		bool		isnull;
-
 		*list = DatumGetInt32(slot_getattr(slot, 1, &isnull));
 		value = slot_getattr(slot, 3, &isnull);
 
@@ -275,8 +254,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 	IndexTuple	itup = NULL;	/* silence compiler warning */
 	int64		inserted = 0;
 
-	TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsMinimalTuple);
-	TupleDesc	tupdesc = buildstate->tupdesc;
+	TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsMinimalTuple);
+	TupleDesc	tupdesc = RelationGetDescr(index);
 
 	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
 
@@ -340,7 +319,6 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 	buildstate->index = index;
 	buildstate->indexInfo = indexInfo;
 	buildstate->typeInfo = IvfflatGetTypeInfo(index);
-	buildstate->tupdesc = RelationGetDescr(index);
 
 	buildstate->lists = IvfflatGetLists(index);
 	buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
@@ -378,24 +356,14 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 				 errmsg("dimensions must be greater than one for this opclass")));
 
 	/* Create tuple description for sorting */
-	buildstate->sortdesc = CreateTemplateTupleDesc(3);
-	TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
-	TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
-	TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
-#if PG_VERSION_NUM >= 190000
-	TupleDescFinalize(buildstate->sortdesc);
-#endif
+	buildstate->tupdesc = CreateTemplateTupleDesc(3);
+	TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
+	TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
+	TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "vector", RelationGetDescr(index)->attrs[0].atttypid, -1, 0);
 
-	buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
+	buildstate->slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsVirtual);
 
-	buildstate->memoryUsed = 0;
-	buildstate->itemsize = buildstate->typeInfo->itemSize(buildstate->dimensions);
-
-	buildstate->memoryUsed += VECTOR_ARRAY_SIZE(buildstate->lists, buildstate->itemsize);
-	IvfflatCheckMemoryUsage(buildstate->memoryUsed);
-	buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
-
-	/* TODO Move allocation to page creation */
+	buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->typeInfo->itemSize(buildstate->dimensions));
 	buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
 
 	buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -438,30 +406,22 @@ ComputeCenters(IvfflatBuildState * buildstate)
 
 	pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
 
+	/* Target 50 samples per list, with at least 10000 samples */
+	/* The number of samples has a large effect on index build time */
+	numSamples = buildstate->lists * 50;
+	if (numSamples < 10000)
+		numSamples = 10000;
+
 	/* Skip samples for unlogged table */
 	if (buildstate->heap == NULL)
 		numSamples = 1;
-	else
-	{
-		int64		maxTuples = (int64) RelationGetNumberOfBlocks(buildstate->heap) * MaxHeapTuplesPerPage;
-
-		/* Target 50 samples per list, with at least 10000 samples */
-		/* The number of samples has a large effect on index build time */
-		numSamples = buildstate->lists * 50;
-		if (numSamples < 10000)
-			numSamples = 10000;
-
-		/* Save memory since will not have more than max tuples */
-		numSamples = Max(Min(numSamples, maxTuples), 1);
-	}
 
 	/* Sample rows */
-	buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize);
-	IvfflatCheckMemoryUsage(buildstate->memoryUsed);
-	buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->itemsize);
+	/* TODO Ensure within maintenance_work_mem */
+	buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
 	if (buildstate->heap != NULL)
 	{
-		IvfflatBench("sample rows", SampleRows(buildstate));
+		SampleRows(buildstate);
 
 		if (buildstate->samples->length < buildstate->lists)
 		{
@@ -473,7 +433,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
 	}
 
 	/* Calculate centers */
-	IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo, buildstate->memoryUsed));
+	IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo));
 
 	/* Free samples before we allocate more memory */
 	VectorArrayFree(buildstate->samples);
@@ -509,8 +469,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum)
  * Create list pages
  */
 static void
-CreateListPages(Relation index, VectorArray centers, int lists,
-				ForkNumber forkNum, ListInfo * *listInfo)
+CreateListPages(Relation index, VectorArray centers, int dimensions,
+				int lists, ForkNumber forkNum, ListInfo * *listInfo)
 {
 	Buffer		buf;
 	Page		page;
@@ -673,14 +633,10 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
 	InitBuildState(&buildstate, ivfspool->heap, ivfspool->index, indexInfo);
 	memcpy(buildstate.centers->items, ivfcenters, buildstate.centers->itemsize * buildstate.centers->maxlen);
 	buildstate.centers->length = buildstate.centers->maxlen;
-	ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate);
+	ivfspool->sortstate = InitBuildSortState(buildstate.tupdesc, sortmem, coordinate);
 	buildstate.sortstate = ivfspool->sortstate;
 	scan = table_beginscan_parallel(ivfspool->heap,
-									ParallelTableScanFromIvfflatShared(ivfshared)
-#if PG_VERSION_NUM >= 190000
-									,SO_NONE
-#endif
-		);
+									ParallelTableScanFromIvfflatShared(ivfshared));
 	reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo,
 									   true, progress, BuildCallback,
 									   (void *) &buildstate, scan);
@@ -994,7 +950,7 @@ AssignTuples(IvfflatBuildState * buildstate)
 	}
 
 	/* Begin serial/leader tuplesort */
-	buildstate->sortstate = InitBuildSortState(buildstate->sortdesc, maintenance_work_mem, coordinate);
+	buildstate->sortstate = InitBuildSortState(buildstate->tupdesc, maintenance_work_mem, coordinate);
 
 	/* Add tuples to sort */
 	if (buildstate->heap != NULL)
@@ -1047,7 +1003,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
 
 	/* Create pages */
 	CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
-	CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo);
+	CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
 	CreateEntryPages(buildstate, forkNum);
 
 	/* Write WAL for initialization fork since GenericXLog functions do not */
@@ -1066,10 +1022,6 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
 	IndexBuildResult *result;
 	IvfflatBuildState buildstate;
 
-#ifdef IVFFLAT_BENCH
-	SeedRandom(42);
-#endif
-
 	BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
 
 	result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));

src/ivfflat.c

@@ -3,35 +3,22 @@
 #include <float.h>
 
 #include "access/amapi.h"
-#include "access/genam.h"
 #include "access/reloptions.h"
 #include "commands/progress.h"
 #include "commands/vacuum.h"
-#include "fmgr.h"
 #include "ivfflat.h"
-#include "nodes/pg_list.h"
 #include "utils/float.h"
 #include "utils/guc.h"
-#include "utils/relcache.h"
 #include "utils/selfuncs.h"
 #include "utils/spccache.h"
-#include "vector.h"
 
 #if PG_VERSION_NUM < 150000
 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
 #endif
 
 int			ivfflat_probes;
-int			ivfflat_iterative_scan;
-int			ivfflat_max_probes;
 static relopt_kind ivfflat_relopt_kind;
 
-static const struct config_enum_entry ivfflat_iterative_scan_options[] = {
-	{"off", IVFFLAT_ITERATIVE_SCAN_OFF, false},
-	{"relaxed_order", IVFFLAT_ITERATIVE_SCAN_RELAXED, false},
-	{NULL, 0, false}
-};
-
 /*
  * Initialize index options and variables
  */
@@ -46,15 +33,6 @@ IvfflatInit(void)
 							"Valid range is 1..lists.", &ivfflat_probes,
 							IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
 
-	DefineCustomEnumVariable("ivfflat.iterative_scan", "Sets the mode for iterative scans",
-							 NULL, &ivfflat_iterative_scan,
-							 IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
-
-	/* If this is less than probes, probes is used */
-	DefineCustomIntVariable("ivfflat.max_probes", "Sets the max number of probes for iterative scans",
-							NULL, &ivfflat_max_probes,
-							IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
-
 	MarkGUCPrefixReserved("ivfflat");
 }
 
@@ -97,17 +75,13 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	Relation	index;
 
 	/* Never use index without order */
-	if (path->indexorderbys == NIL)
+	if (path->indexorderbys == NULL)
 	{
 		*indexStartupCost = get_float8_infinity();
 		*indexTotalCost = get_float8_infinity();
 		*indexSelectivity = 0;
 		*indexCorrelation = 0;
 		*indexPages = 0;
-#if PG_VERSION_NUM >= 180000
-		/* See "On disable_cost" thread on pgsql-hackers */
-		path->path.disabled_nodes = 2;
-#endif
 		return;
 	}
 
@@ -184,64 +158,6 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
 Datum
 ivfflathandler(PG_FUNCTION_ARGS)
 {
-#if PG_VERSION_NUM >= 190000
-	static const IndexAmRoutine amroutine = {
-		.type = T_IndexAmRoutine,
-		.amstrategies = 0,
-		.amsupport = 5,
-		.amoptsprocnum = 0,
-		.amcanorder = false,
-		.amcanorderbyop = true,
-		.amcanhash = false,
-		.amconsistentequality = false,
-		.amconsistentordering = false,
-		.amcanbackward = false,
-		.amcanunique = false,
-		.amcanmulticol = false,
-		.amoptionalkey = true,
-		.amsearcharray = false,
-		.amsearchnulls = false,
-		.amstorage = false,
-		.amclusterable = false,
-		.ampredlocks = false,
-		.amcanparallel = false,
-		.amcanbuildparallel = true,
-		.amcaninclude = false,
-		.amusemaintenanceworkmem = false,
-		.amsummarizing = false,
-		.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
-		.amkeytype = InvalidOid,
-
-		.ambuild = ivfflatbuild,
-		.ambuildempty = ivfflatbuildempty,
-		.aminsert = ivfflatinsert,
-		.aminsertcleanup = NULL,
-		.ambulkdelete = ivfflatbulkdelete,
-		.amvacuumcleanup = ivfflatvacuumcleanup,
-		.amcanreturn = NULL,
-		.amcostestimate = ivfflatcostestimate,
-		.amgettreeheight = NULL,
-		.amoptions = ivfflatoptions,
-		.amproperty = NULL,
-		.ambuildphasename = ivfflatbuildphasename,
-		.amvalidate = ivfflatvalidate,
-		.amadjustmembers = NULL,
-		.ambeginscan = ivfflatbeginscan,
-		.amrescan = ivfflatrescan,
-		.amgettuple = ivfflatgettuple,
-		.amgetbitmap = NULL,
-		.amendscan = ivfflatendscan,
-		.ammarkpos = NULL,
-		.amrestrpos = NULL,
-		.amestimateparallelscan = NULL,
-		.aminitparallelscan = NULL,
-		.amparallelrescan = NULL,
-		.amtranslatestrategy = NULL,
-		.amtranslatecmptype = NULL,
-	};
-
-	PG_RETURN_POINTER(&amroutine);
-#else
 	IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
 
 	amroutine->amstrategies = 0;
@@ -249,11 +165,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
 	amroutine->amoptsprocnum = 0;
 	amroutine->amcanorder = false;
 	amroutine->amcanorderbyop = true;
-#if PG_VERSION_NUM >= 180000
-	amroutine->amcanhash = false;
-	amroutine->amconsistentequality = false;
-	amroutine->amconsistentordering = false;
-#endif
 	amroutine->amcanbackward = false;	/* can change direction mid-scan */
 	amroutine->amcanunique = false;
 	amroutine->amcanmulticol = false;
@@ -286,9 +197,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
 	amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
 	amroutine->amcanreturn = NULL;	/* tuple not included in heapsort */
 	amroutine->amcostestimate = ivfflatcostestimate;
-#if PG_VERSION_NUM >= 180000
-	amroutine->amgettreeheight = NULL;
-#endif
 	amroutine->amoptions = ivfflatoptions;
 	amroutine->amproperty = NULL;	/* TODO AMPROP_DISTANCE_ORDERABLE */
 	amroutine->ambuildphasename = ivfflatbuildphasename;
@@ -309,11 +217,5 @@ ivfflathandler(PG_FUNCTION_ARGS)
 	amroutine->aminitparallelscan = NULL;
 	amroutine->amparallelrescan = NULL;
 
-#if PG_VERSION_NUM >= 180000
-	amroutine->amtranslatestrategy = NULL;
-	amroutine->amtranslatecmptype = NULL;
-#endif
-
 	PG_RETURN_POINTER(amroutine);
-#endif
 }

src/ivfflat.h

@@ -9,15 +9,10 @@
 #include "lib/pairingheap.h"
 #include "nodes/execnodes.h"
 #include "port.h"				/* for random() */
-#include "storage/condition_variable.h"
 #include "utils/sampling.h"
 #include "utils/tuplesort.h"
 #include "vector.h"
 
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
-
 #if PG_VERSION_NUM >= 150000
 #include "common/pg_prng.h"
 #endif
@@ -26,10 +21,6 @@
 #include "portability/instr_time.h"
 #endif
 
-#if PG_VERSION_NUM >= 190000
-typedef Pointer Item;
-#endif
-
 #define IVFFLAT_MAX_DIM 2000
 
 /* Support functions */
@@ -82,23 +73,13 @@ typedef Pointer Item;
 #if PG_VERSION_NUM >= 150000
 #define RandomDouble() pg_prng_double(&pg_global_prng_state)
 #define RandomInt() pg_prng_uint32(&pg_global_prng_state)
-#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
 #else
 #define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
 #define RandomInt() random()
-#define SeedRandom(seed) srandom(seed)
 #endif
 
 /* Variables */
 extern int	ivfflat_probes;
-extern int	ivfflat_iterative_scan;
-extern int	ivfflat_max_probes;
-
-typedef enum IvfflatIterativeScanMode
-{
-	IVFFLAT_ITERATIVE_SCAN_OFF,
-	IVFFLAT_ITERATIVE_SCAN_RELAXED
-}			IvfflatIterativeScanMode;
 
 typedef struct VectorArrayData
 {
@@ -184,7 +165,6 @@ typedef struct IvfflatBuildState
 	Relation	index;
 	IndexInfo  *indexInfo;
 	const		IvfflatTypeInfo *typeInfo;
-	TupleDesc	tupdesc;
 
 	/* Settings */
 	int			dimensions;
@@ -204,7 +184,6 @@ typedef struct IvfflatBuildState
 	VectorArray samples;
 	VectorArray centers;
 	ListInfo   *listInfo;
-	Size		itemsize;
 
 #ifdef IVFFLAT_KMEANS_DEBUG
 	double		inertia;
@@ -215,16 +194,14 @@ typedef struct IvfflatBuildState
 	/* Sampling */
 	BlockSamplerData bs;
 	ReservoirStateData rstate;
-	double		samplerows;
-	double		rowstoskip;
+	int			rowstoskip;
 
 	/* Sorting */
 	Tuplesortstate *sortstate;
-	TupleDesc	sortdesc;
+	TupleDesc	tupdesc;
 	TupleTableSlot *slot;
 
 	/* Memory */
-	Size		memoryUsed;
 	MemoryContext tmpCtx;
 
 	/* Parallel builds */
@@ -270,11 +247,8 @@ typedef struct IvfflatScanOpaqueData
 {
 	const		IvfflatTypeInfo *typeInfo;
 	int			probes;
-	int			maxProbes;
 	int			dimensions;
 	bool		first;
-	Datum		value;
-	MemoryContext tmpCtx;
 
 	/* Sorting */
 	Tuplesortstate *sortstate;
@@ -291,9 +265,7 @@ typedef struct IvfflatScanOpaqueData
 
 	/* Lists */
 	pairingheap *listQueue;
-	BlockNumber *listPages;
-	int			listIndex;
-	IvfflatScanList *lists;
+	IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER];	/* must come last */
 }			IvfflatScanOpaqueData;
 
 typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
@@ -305,32 +277,22 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
 static inline Pointer
 VectorArrayGet(VectorArray arr, int offset)
 {
-	if (offset >= arr->maxlen)
-		elog(ERROR, "safety check failed");
-
 	return ((char *) arr->items) + (offset * arr->itemsize);
 }
 
 static inline void
 VectorArraySet(VectorArray arr, int offset, Pointer val)
 {
-	Size		size = VARSIZE_ANY(val);
-
-	if (size > arr->itemsize)
-		elog(ERROR, "safety check failed");
-
-	memcpy(VectorArrayGet(arr, offset), val, size);
+	memcpy(VectorArrayGet(arr, offset), val, VARSIZE_ANY(val));
 }
 
 /* Methods */
 VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
 void		VectorArrayFree(VectorArray arr);
-void		IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed);
+void		IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo);
 FmgrInfo   *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
 Datum		IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
 bool		IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
-void		IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx);
-void		IvfflatCheckMemoryUsage(Size totalSize);
 int			IvfflatGetLists(Relation index);
 void		IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
 void		IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);

src/ivfinsert.c

@@ -2,16 +2,11 @@
 
 #include <float.h>
 
-#include "access/genam.h"
 #include "access/generic_xlog.h"
-#include "access/itup.h"
-#include "fmgr.h"
 #include "ivfflat.h"
-#include "nodes/execnodes.h"
 #include "storage/bufmgr.h"
 #include "storage/lmgr.h"
 #include "utils/memutils.h"
-#include "utils/rel.h"
 
 /*
  * Find the list that minimizes the distance function
@@ -70,7 +65,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
  * Insert a tuple into the index
  */
 static void
-InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
+InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
 {
 	const		IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
 	IndexTuple	itup;
@@ -103,7 +98,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
 	IvfflatGetMetaPageInfo(index, NULL, NULL);
 
 	/* Find the insert page - sets the page and list info */
-	FindInsertPage(index, &value, &insertPage, &listInfo);
+	FindInsertPage(index, values, &insertPage, &listInfo);
 	Assert(BlockNumberIsValid(insertPage));
 	originalInsertPage = insertPage;
 
@@ -209,7 +204,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
 	oldCtx = MemoryContextSwitchTo(insertCtx);
 
 	/* Insert tuple */
-	InsertTuple(index, values, isnull, heap_tid);
+	InsertTuple(index, values, isnull, heap_tid, heap);
 
 	/* Delete memory context */
 	MemoryContextSwitchTo(oldCtx);

src/ivfkmeans.c

@@ -1,19 +1,17 @@
 #include "postgres.h"
 
 #include <float.h>
-#include <limits.h>
 #include <math.h>
 
-#include "access/genam.h"
-#include "fmgr.h"
+#include "bitvec.h"
+#include "halfutils.h"
+#include "halfvec.h"
 #include "ivfflat.h"
 #include "miscadmin.h"
+#include "utils/builtins.h"
+#include "utils/datum.h"
 #include "utils/memutils.h"
-#include "utils/relcache.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
+#include "vector.h"
 
 /*
  * Initialize with kmeans++
@@ -99,8 +97,22 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
 	MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
 												  "Ivfflat norm temporary context",
 												  ALLOCSET_DEFAULT_SIZES);
+	MemoryContext oldCtx = MemoryContextSwitchTo(normCtx);
 
-	IvfflatNormVectors(typeInfo, collation, centers, normCtx);
+	for (int j = 0; j < centers->length; j++)
+	{
+		Datum		center = PointerGetDatum(VectorArrayGet(centers, j));
+		Datum		newCenter = IvfflatNormValue(typeInfo, collation, center);
+		Size		size = VARSIZE_ANY(DatumGetPointer(newCenter));
+
+		if (size > centers->itemsize)
+			elog(ERROR, "safety check failed");
+
+		memcpy(DatumGetPointer(center), DatumGetPointer(newCenter), size);
+		MemoryContextReset(normCtx);
+	}
+
+	MemoryContextSwitchTo(oldCtx);
 	MemoryContextDelete(normCtx);
 }
 
@@ -244,7 +256,7 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
  * https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
  */
 static void
-ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
+ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
 {
 	FmgrInfo   *procinfo;
 	FmgrInfo   *normprocinfo;
@@ -263,6 +275,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
 	float	   *newcdist;
 
 	/* Calculate allocation sizes */
+	Size		samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->itemsize);
+	Size		centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->itemsize);
 	Size		newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
 	Size		aggSize = sizeof(float) * (int64) numCenters * dimensions;
 	Size		centerCountsSize = sizeof(int) * numCenters;
@@ -274,13 +288,18 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
 	Size		newcdistSize = sizeof(float) * numCenters;
 
 	/* Calculate total size */
-	Size		totalSize = memoryUsed + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
+	Size		totalSize = samplesSize + centersSize + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
 
 	/* Check memory requirements */
-	IvfflatCheckMemoryUsage(totalSize);
+	/* Add one to error message to ceil */
+	if (totalSize > (Size) maintenance_work_mem * 1024L)
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
+						totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
 
 	/* Ensure indexing does not overflow */
-	if (numCenters > INT_MAX / numCenters)
+	if (numCenters * numCenters > INT_MAX)
 		elog(ERROR, "Indexing overflow detected. Please report a bug.");
 
 	/* Set support functions */
@@ -541,7 +560,7 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
  * We use spherical k-means for inner product and cosine
  */
 void
-IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
+IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
 {
 	MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
 													"Ivfflat kmeans temporary context",
@@ -551,7 +570,7 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
 	if (samples->length == 0)
 		RandomCenters(index, centers, typeInfo);
 	else
-		ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);
+		ElkanKmeans(index, samples, centers, typeInfo);
 
 	CheckCenters(index, centers, typeInfo);
 

src/ivfscan.c

@@ -2,25 +2,17 @@
 
 #include <float.h>
 
-#include "access/genam.h"
-#include "access/itup.h"
 #include "access/relscan.h"
-#include "access/tupdesc.h"
 #include "catalog/pg_operator_d.h"
 #include "catalog/pg_type_d.h"
-#include "fmgr.h"
 #include "lib/pairingheap.h"
 #include "ivfflat.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "storage/bufmgr.h"
-#include "utils/memutils.h"
-#include "utils/rel.h"
-#include "utils/snapmgr.h"
-#include "utils/tuplesort.h"
 
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
+#ifdef IVFFLAT_MEMORY
+#include "utils/memutils.h"
 #endif
 
 #define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
@@ -73,7 +65,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
 			/* Use procinfo from the index instead of scan key for performance */
 			distance = DatumGetFloat8(so->distfunc(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
 
-			if (listCount < so->maxProbes)
+			if (listCount < so->probes)
 			{
 				IvfflatScanList *scanlist;
 
@@ -86,7 +78,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
 				pairingheap_add(so->listQueue, &scanlist->ph_node);
 
 				/* Calculate max distance */
-				if (listCount == so->maxProbes)
+				if (listCount == so->probes)
 					maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
 			}
 			else if (distance < maxDistance)
@@ -110,11 +102,6 @@ GetScanLists(IndexScanDesc scan, Datum value)
 
 		UnlockReleaseBuffer(cbuf);
 	}
-
-	for (int i = listCount - 1; i >= 0; i--)
-		so->listPages[i] = GetScanList(pairingheap_remove_first(so->listQueue))->startPage;
-
-	Assert(pairingheap_is_empty(so->listQueue));
 }
 
 /*
@@ -125,15 +112,13 @@ GetScanItems(IndexScanDesc scan, Datum value)
 {
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
 	TupleDesc	tupdesc = RelationGetDescr(scan->indexRelation);
+	double		tuples = 0;
 	TupleTableSlot *slot = so->vslot;
-	int			batchProbes = 0;
-
-	tuplesort_reset(so->sortstate);
 
 	/* Search closest probes lists */
-	while (so->listIndex < so->maxProbes && (++batchProbes) <= so->probes)
+	while (!pairingheap_is_empty(so->listQueue))
 	{
-		BlockNumber searchPage = so->listPages[so->listIndex++];
+		BlockNumber searchPage = GetScanList(pairingheap_remove_first(so->listQueue))->startPage;
 
 		/* Search all entry pages for list */
 		while (BlockNumberIsValid(searchPage))
@@ -171,6 +156,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
 				ExecStoreVirtualTuple(slot);
 
 				tuplesort_puttupleslot(so->sortstate, slot);
+
+				tuples++;
 			}
 
 			searchPage = IvfflatPageGetOpaque(page)->nextblkno;
@@ -179,11 +166,13 @@ GetScanItems(IndexScanDesc scan, Datum value)
 		}
 	}
 
-	tuplesort_performsort(so->sortstate);
+	if (tuples < 100)
+		ereport(DEBUG1,
+				(errmsg("index scan found few tuples"),
+				 errdetail("Index may have been created with little data."),
+				 errhint("Recreate the index and possibly decrease lists.")));
 
-#if defined(IVFFLAT_MEMORY)
-	elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
-#endif
+	tuplesort_performsort(so->sortstate);
 }
 
 /*
@@ -220,13 +209,7 @@ GetScanValue(IndexScanDesc scan)
 
 		/* Normalize if needed */
 		if (so->normprocinfo != NULL)
-		{
-			MemoryContext oldCtx = MemoryContextSwitchTo(so->tmpCtx);
-
 			value = IvfflatNormValue(so->typeInfo, so->collation, value);
-
-			MemoryContextSwitchTo(oldCtx);
-		}
 	}
 
 	return value;
@@ -257,30 +240,19 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	int			lists;
 	int			dimensions;
 	int			probes = ivfflat_probes;
-	int			maxProbes;
-	MemoryContext oldCtx;
 
 	scan = RelationGetIndexScan(index, nkeys, norderbys);
 
 	/* Get lists and dimensions from metapage */
 	IvfflatGetMetaPageInfo(index, &lists, &dimensions);
 
-	if (ivfflat_iterative_scan != IVFFLAT_ITERATIVE_SCAN_OFF)
-		maxProbes = Max(ivfflat_max_probes, probes);
-	else
-		maxProbes = probes;
-
 	if (probes > lists)
 		probes = lists;
 
-	if (maxProbes > lists)
-		maxProbes = lists;
-
-	so = (IvfflatScanOpaque) palloc(sizeof(IvfflatScanOpaqueData));
+	so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + probes * sizeof(IvfflatScanList));
 	so->typeInfo = IvfflatGetTypeInfo(index);
 	so->first = true;
 	so->probes = probes;
-	so->maxProbes = maxProbes;
 	so->dimensions = dimensions;
 
 	/* Set support functions */
@@ -288,19 +260,10 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	so->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
 	so->collation = index->rd_indcollation[0];
 
-	so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
-									   "Ivfflat scan temporary context",
-									   ALLOCSET_DEFAULT_SIZES);
-
-	oldCtx = MemoryContextSwitchTo(so->tmpCtx);
-
 	/* Create tuple description for sorting */
 	so->tupdesc = CreateTemplateTupleDesc(2);
 	TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
 	TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
-#if PG_VERSION_NUM >= 190000
-	TupleDescFinalize(so->tupdesc);
-#endif
 
 	/* Prep sort */
 	so->sortstate = InitScanSortState(so->tupdesc);
@@ -317,11 +280,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	so->bas = GetAccessStrategy(BAS_BULKREAD);
 
 	so->listQueue = pairingheap_allocate(CompareLists, scan);
-	so->listPages = palloc(maxProbes * sizeof(BlockNumber));
-	so->listIndex = 0;
-	so->lists = palloc(maxProbes * sizeof(IvfflatScanList));
-
-	MemoryContextSwitchTo(oldCtx);
 
 	scan->opaque = so;
 
@@ -336,9 +294,11 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
 {
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
 
+	if (!so->first)
+		tuplesort_reset(so->sortstate);
+
 	so->first = true;
 	pairingheap_reset(so->listQueue);
-	so->listIndex = 0;
 
 	if (keys && scan->numberOfKeys > 0)
 		memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
@@ -354,8 +314,6 @@ bool
 ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
 {
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
-	ItemPointer heaptid;
-	bool		isnull;
 
 	/*
 	 * Index can be used to scan backward, but Postgres doesn't support
@@ -369,10 +327,6 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
 
 		/* Count index scan for stats */
 		pgstat_count_index_scan(scan->indexRelation);
-#if PG_VERSION_NUM >= 180000
-		if (scan->instrument)
-			scan->instrument->nsearches++;
-#endif
 
 		/* Safety check */
 		if (scan->orderByData == NULL)
@@ -387,23 +341,28 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
 		IvfflatBench("GetScanLists", GetScanLists(scan, value));
 		IvfflatBench("GetScanItems", GetScanItems(scan, value));
 		so->first = false;
-		so->value = value;
+
+#if defined(IVFFLAT_MEMORY)
+		elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
+#endif
+
+		/* Clean up if we allocated a new value */
+		if (value != scan->orderByData->sk_argument)
+			pfree(DatumGetPointer(value));
 	}
 
-	while (!tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
+	if (tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
 	{
-		if (so->listIndex == so->maxProbes)
-			return false;
+		bool		isnull;
+		ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
 
-		IvfflatBench("GetScanItems", GetScanItems(scan, so->value));
+		scan->xs_heaptid = *heaptid;
+		scan->xs_recheck = false;
+		scan->xs_recheckorderby = false;
+		return true;
 	}
 
-	heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
-
-	scan->xs_heaptid = *heaptid;
-	scan->xs_recheck = false;
-	scan->xs_recheckorderby = false;
-	return true;
+	return false;
 }
 
 /*
@@ -414,10 +373,12 @@ ivfflatendscan(IndexScanDesc scan)
 {
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
 
-	/* Free any temporary files */
+	pairingheap_free(so->listQueue);
 	tuplesort_end(so->sortstate);
+	FreeAccessStrategy(so->bas);
+	FreeTupleDesc(so->tupdesc);
 
-	MemoryContextDelete(so->tmpCtx);
+	/* TODO Free vslot and mslot without freeing TupleDesc */
 
 	pfree(so);
 	scan->opaque = NULL;

src/ivfutils.c

@@ -1,21 +1,13 @@
 #include "postgres.h"
 
-#include "access/genam.h"
 #include "access/generic_xlog.h"
+#include "bitvec.h"
+#include "catalog/pg_type.h"
 #include "fmgr.h"
 #include "halfutils.h"
 #include "halfvec.h"
 #include "ivfflat.h"
-#include "miscadmin.h"
 #include "storage/bufmgr.h"
-#include "utils/memutils.h"
-#include "utils/relcache.h"
-#include "utils/varbit.h"
-#include "vector.h"
-
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
 
 /*
  * Allocate a vector array
@@ -23,15 +15,11 @@
 VectorArray
 VectorArrayInit(int maxlen, int dimensions, Size itemsize)
 {
-	VectorArray res;
-
-	if (maxlen < 1 || dimensions < 1)
-		elog(ERROR, "safety check failed");
+	VectorArray res = palloc(sizeof(VectorArrayData));
 
 	/* Ensure items are aligned to prevent UB */
 	itemsize = MAXALIGN(itemsize);
 
-	res = palloc(sizeof(VectorArrayData));
 	res->length = 0;
 	res->maxlen = maxlen;
 	res->dim = dimensions;
@@ -94,40 +82,6 @@ IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
 	return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
 }
 
-/*
- * Normalize vectors
- */
-void
-IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx)
-{
-	MemoryContext oldCtx = MemoryContextSwitchTo(tmpCtx);
-
-	for (int i = 0; i < arr->length; i++)
-	{
-		Datum		value = PointerGetDatum(VectorArrayGet(arr, i));
-		Datum		newValue = IvfflatNormValue(typeInfo, collation, value);
-
-		VectorArraySet(arr, i, DatumGetPointer(newValue));
-		MemoryContextReset(tmpCtx);
-	}
-
-	MemoryContextSwitchTo(oldCtx);
-}
-
-/*
- * Check memory usage
- */
-void
-IvfflatCheckMemoryUsage(Size totalSize)
-{
-	/* Add one to error message to ceil */
-	if (totalSize > maintenance_work_mem * (Size) 1024)
-		ereport(ERROR,
-				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-				 errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
-						totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
-}
-
 /*
  * New buffer
  */
@@ -305,8 +259,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x)
 	SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
 	vec->dim = dimensions;
 
-	for (int i = 0; i < dimensions; i++)
-		vec->x[i] = x[i];
+	for (int k = 0; k < dimensions; k++)
+		vec->x[k] = x[k];
 }
 
 static void
@@ -317,8 +271,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
 	SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
 	vec->dim = dimensions;
 
-	for (int i = 0; i < dimensions; i++)
-		vec->x[i] = Float4ToHalfUnchecked(x[i]);
+	for (int k = 0; k < dimensions; k++)
+		vec->x[k] = Float4ToHalfUnchecked(x[k]);
 }
 
 static void
@@ -330,33 +284,29 @@ BitUpdateCenter(Pointer v, int dimensions, float *x)
 	SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
 	VARBITLEN(vec) = dimensions;
 
-	for (uint32 i = 0; i < VARBITBYTES(vec); i++)
-		nx[i] = 0;
+	for (uint32 k = 0; k < VARBITBYTES(vec); k++)
+		nx[k] = 0;
 
-	for (int i = 0; i < dimensions; i++)
-		nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8));
+	for (int k = 0; k < dimensions; k++)
+		nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
 }
 
 static void
 VectorSumCenter(Pointer v, float *x)
 {
 	Vector	   *vec = (Vector *) v;
-	int			dim = vec->dim;
 
-	/* Auto-vectorized */
-	for (int i = 0; i < dim; i++)
-		x[i] += vec->x[i];
+	for (int k = 0; k < vec->dim; k++)
+		x[k] += vec->x[k];
 }
 
 static void
 HalfvecSumCenter(Pointer v, float *x)
 {
 	HalfVector *vec = (HalfVector *) v;
-	int			dim = vec->dim;
 
-	/* Auto-vectorized on aarch64 */
-	for (int i = 0; i < dim; i++)
-		x[i] += HalfToFloat4(vec->x[i]);
+	for (int k = 0; k < vec->dim; k++)
+		x[k] += HalfToFloat4(vec->x[k]);
 }
 
 static void
@@ -364,8 +314,8 @@ BitSumCenter(Pointer v, float *x)
 {
 	VarBit	   *vec = (VarBit *) v;
 
-	for (int i = 0; i < VARBITLEN(vec); i++)
-		x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01);
+	for (int k = 0; k < VARBITLEN(vec); k++)
+		x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
 }
 
 /*
@@ -405,7 +355,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
-}
+};
 
 FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
 Datum
@@ -420,4 +370,4 @@ ivfflat_bit_support(PG_FUNCTION_ARGS)
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
-}
+};

src/ivfvacuum.c

@@ -1,16 +1,9 @@
 #include "postgres.h"
 
-#include "access/genam.h"
 #include "access/generic_xlog.h"
-#include "access/itup.h"
 #include "commands/vacuum.h"
 #include "ivfflat.h"
 #include "storage/bufmgr.h"
-#include "utils/relcache.h"
-
-#if PG_VERSION_NUM >= 180000
-#define vacuum_delay_point() vacuum_delay_point(false)
-#endif
 
 /*
  * Bulk delete tuples from the index
@@ -33,7 +26,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 		Page		cpage;
 		OffsetNumber coffno;
 		OffsetNumber cmaxoffno;
-		BlockNumber listPages[MaxOffsetNumber];
+		BlockNumber startPages[MaxOffsetNumber];
 		ListInfo	listInfo;
 
 		cbuf = ReadBuffer(index, blkno);
@@ -47,7 +40,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 		{
 			IvfflatList list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, coffno));
 
-			listPages[coffno - FirstOffsetNumber] = list->startPage;
+			startPages[coffno - FirstOffsetNumber] = list->startPage;
 		}
 
 		listInfo.blkno = blkno;
@@ -57,7 +50,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 
 		for (coffno = FirstOffsetNumber; coffno <= cmaxoffno; coffno = OffsetNumberNext(coffno))
 		{
-			BlockNumber searchPage = listPages[coffno - FirstOffsetNumber];
+			BlockNumber searchPage = startPages[coffno - FirstOffsetNumber];
 			BlockNumber insertPage = InvalidBlockNumber;
 
 			/* Iterate over entry pages */

src/sparsevec.c

@@ -4,26 +4,23 @@
 #include <math.h>
 
 #include "catalog/pg_type.h"
-#include "common/shortest_dec.h"
+#include "common/string.h"
 #include "fmgr.h"
 #include "halfutils.h"
 #include "halfvec.h"
-#include "lib/stringinfo.h"
 #include "libpq/pqformat.h"
 #include "sparsevec.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
-#include "utils/float.h"
-#include "utils/fmgrprotos.h"
 #include "utils/lsyscache.h"
 #include "vector.h"
 
-#if PG_VERSION_NUM >= 160000
-#include "varatt.h"
-#endif
-
-#if PG_VERSION_NUM >= 170000
-#include "parser/scansup.h"
+#if PG_VERSION_NUM >= 120000
+#include "common/shortest_dec.h"
+#include "utils/float.h"
+#else
+#include <float.h>
+#include "utils/builtins.h"
 #endif
 
 typedef struct SparseInputElement
@@ -159,9 +156,9 @@ InitSparseVector(int dim, int nnz)
 	return result;
 }
 
-#if PG_VERSION_NUM >= 170000
-#define sparsevec_isspace(ch) scanner_isspace(ch)
-#else
+/*
+ * Check for whitespace, since array_isspace() is static
+ */
 static inline bool
 sparsevec_isspace(char ch)
 {
@@ -174,7 +171,6 @@ sparsevec_isspace(char ch)
 		return true;
 	return false;
 }
-#endif
 
 /*
  * Compare indices
@@ -182,10 +178,10 @@ sparsevec_isspace(char ch)
 static int
 CompareIndices(const void *a, const void *b)
 {
-	if (((const SparseInputElement *) a)->index < ((const SparseInputElement *) b)->index)
+	if (((SparseInputElement *) a)->index < ((SparseInputElement *) b)->index)
 		return -1;
 
-	if (((const SparseInputElement *) a)->index > ((const SparseInputElement *) b)->index)
+	if (((SparseInputElement *) a)->index > ((SparseInputElement *) b)->index)
 		return 1;
 
 	return 0;

src/vector.c

@@ -16,20 +16,16 @@
 #include "port.h"				/* for strtof() */
 #include "sparsevec.h"
 #include "utils/array.h"
+#include "utils/builtins.h"
 #include "utils/float.h"
-#include "utils/fmgrprotos.h"
 #include "utils/lsyscache.h"
-#include "utils/varbit.h"
+#include "utils/numeric.h"
 #include "vector.h"
 
 #if PG_VERSION_NUM >= 160000
 #include "varatt.h"
 #endif
 
-#if PG_VERSION_NUM >= 170000
-#include "parser/scansup.h"
-#endif
-
 #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
 #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
 
@@ -39,11 +35,7 @@
 #define VECTOR_TARGET_CLONES
 #endif
 
-#if PG_VERSION_NUM >= 180000
-PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.5");
-#else
 PG_MODULE_MAGIC;
-#endif
 
 /*
  * Initialize index options and variables
@@ -133,9 +125,9 @@ InitVector(int dim)
 	return result;
 }
 
-#if PG_VERSION_NUM >= 170000
-#define vector_isspace(ch) scanner_isspace(ch)
-#else
+/*
+ * Check for whitespace, since array_isspace() is static
+ */
 static inline bool
 vector_isspace(char ch)
 {
@@ -148,7 +140,6 @@ vector_isspace(char ch)
 		return true;
 	return false;
 }
-#endif
 
 /*
  * Check state array
@@ -929,13 +920,11 @@ vector_concat(PG_FUNCTION_ARGS)
 	CheckDim(dim);
 	result = InitVector(dim);
 
-	/* Auto-vectorized */
-	for (int i = 0, imax = a->dim; i < imax; i++)
+	for (int i = 0; i < a->dim; i++)
 		result->x[i] = a->x[i];
 
-	/* Auto-vectorized */
-	for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++)
-		result->x[i + start] = b->x[i];
+	for (int i = 0; i < b->dim; i++)
+		result->x[i + a->dim] = b->x[i];
 
 	PG_RETURN_POINTER(result);
 }
@@ -951,21 +940,8 @@ binary_quantize(PG_FUNCTION_ARGS)
 	float	   *ax = a->x;
 	VarBit	   *result = InitBitVector(a->dim);
 	unsigned char *rx = VARBITS(result);
-	int			i = 0;
-	int			count = (a->dim / 8) * 8;
 
-	/* Auto-vectorized */
-	for (; i < count; i += 8)
-	{
-		unsigned char result_byte = 0;
-
-		for (int j = 0; j < 8; j++)
-			result_byte |= (ax[i + j] > 0) << (7 - j);
-
-		rx[i / 8] = result_byte;
-	}
-
-	for (; i < a->dim; i++)
+	for (int i = 0; i < a->dim; i++)
 		rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
 
 	PG_RETURN_VARBIT_P(result);

test/expected/halfvec.out

@@ -540,12 +540,6 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec)
  01001110101
 (1 row)
 
-SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
-   binary_quantize   
----------------------
- 1110110110011011011
-(1 row)
-
 SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
  subvector 
 -----------

test/expected/hnsw_bit.out

@@ -49,11 +49,3 @@ CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
 CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
 ERROR:  column cannot have more than 64000 dimensions for hnsw index
 DROP TABLE t;
--- dimensions
-CREATE TABLE t (val bit(64000));
-CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
-DROP TABLE t;
-CREATE TABLE t (val bit(64001));
-CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
-ERROR:  column cannot have more than 64000 dimensions for hnsw index
-DROP TABLE t;

test/expected/hnsw_halfvec.out

@@ -100,11 +100,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t
 (1 row)
 
 DROP TABLE t;
--- dimensions
-CREATE TABLE t (val halfvec(4000));
-CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
-DROP TABLE t;
-CREATE TABLE t (val halfvec(4001));
-CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
-ERROR:  column cannot have more than 4000 dimensions for hnsw index
-DROP TABLE t;

test/expected/hnsw_vector.out

@@ -99,38 +99,6 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2
      4
 (1 row)
 
-DROP TABLE t;
--- iterative
-CREATE TABLE t (val vector(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING hnsw (val vector_l2_ops);
-SET hnsw.iterative_scan = strict_order;
-SET hnsw.ef_search = 1;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-   val   
----------
- [1,2,3]
- [1,1,1]
- [0,0,0]
-(3 rows)
-
-SET hnsw.iterative_scan = relaxed_order;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-   val   
----------
- [1,2,3]
- [1,1,1]
- [0,0,0]
-(3 rows)
-
-TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
- val 
------
-(0 rows)
-
-RESET hnsw.iterative_scan;
-RESET hnsw.ef_search;
 DROP TABLE t;
 -- unlogged
 CREATE UNLOGGED TABLE t (val vector(3));
@@ -161,7 +129,6 @@ ERROR:  value 1001 out of bounds for option "ef_construction"
 DETAIL:  Valid values are between "4" and "1000".
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
 ERROR:  ef_construction must be greater than or equal to 2 * m
-DROP TABLE t;
 SHOW hnsw.ef_search;
  hnsw.ef_search 
 ----------------
@@ -172,38 +139,4 @@ SET hnsw.ef_search = 0;
 ERROR:  0 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
 SET hnsw.ef_search = 1001;
 ERROR:  1001 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
-SHOW hnsw.iterative_scan;
- hnsw.iterative_scan 
----------------------
- off
-(1 row)
-
-SET hnsw.iterative_scan = on;
-ERROR:  invalid value for parameter "hnsw.iterative_scan": "on"
-HINT:  Available values: off, relaxed_order, strict_order.
-SHOW hnsw.max_scan_tuples;
- hnsw.max_scan_tuples 
-----------------------
- 20000
-(1 row)
-
-SET hnsw.max_scan_tuples = 0;
-ERROR:  0 is outside the valid range for parameter "hnsw.max_scan_tuples" (1 .. 2147483647)
-SHOW hnsw.scan_mem_multiplier;
- hnsw.scan_mem_multiplier 
---------------------------
- 1
-(1 row)
-
-SET hnsw.scan_mem_multiplier = 0;
-ERROR:  0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
-SET hnsw.scan_mem_multiplier = 1001;
-ERROR:  1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
--- dimensions
-CREATE TABLE t (val vector(2000));
-CREATE INDEX ON t USING hnsw (val vector_l2_ops);
-DROP TABLE t;
-CREATE TABLE t (val vector(2001));
-CREATE INDEX ON t USING hnsw (val vector_l2_ops);
-ERROR:  column cannot have more than 2000 dimensions for hnsw index
 DROP TABLE t;

test/expected/ivfflat_bit.out

@@ -35,32 +35,3 @@ NOTICE:  ivfflat index created with little data
 DETAIL:  This will cause low recall.
 HINT:  Drop the index until the table has more data.
 DROP TABLE t;
--- dimensions
-CREATE TABLE t (val bit(64000));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-CREATE TABLE t (val bit(64001));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-ERROR:  column cannot have more than 64000 dimensions for ivfflat index
-DROP TABLE t;
--- memory
-SET maintenance_work_mem = '1MB';
-CREATE TABLE t (val bit(64000));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-RESET maintenance_work_mem;
-SET maintenance_work_mem = '29MB';
-CREATE TABLE t (val bit(64000));
-INSERT INTO t (val) VALUES (B'0'::bit(64000));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-RESET maintenance_work_mem;

test/expected/ivfflat_halfvec.out

@@ -82,32 +82,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t
 (1 row)
 
 DROP TABLE t;
--- dimensions
-CREATE TABLE t (val halfvec(4000));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-CREATE TABLE t (val halfvec(4001));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-ERROR:  column cannot have more than 4000 dimensions for ivfflat index
-DROP TABLE t;
--- memory
-SET maintenance_work_mem = '1MB';
-CREATE TABLE t (val halfvec(4000));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-RESET maintenance_work_mem;
-SET maintenance_work_mem = '6MB';
-CREATE TABLE t (val halfvec(4000));
-INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-RESET maintenance_work_mem;

test/expected/ivfflat_vector.out

@@ -81,46 +81,6 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2
      3
 (1 row)
 
-DROP TABLE t;
--- iterative
-CREATE TABLE t (val vector(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 3);
-SET ivfflat.iterative_scan = relaxed_order;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-   val   
----------
- [1,2,3]
- [1,1,1]
- [0,0,0]
-(3 rows)
-
-SET ivfflat.max_probes = 1;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-   val   
----------
- [1,2,3]
-(1 row)
-
-SET ivfflat.max_probes = 2;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-   val   
----------
- [1,2,3]
- [1,1,1]
-(2 rows)
-
-TRUNCATE t;
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
- val 
------
-(0 rows)
-
-RESET ivfflat.iterative_scan;
-RESET ivfflat.max_probes;
 DROP TABLE t;
 -- unlogged
 CREATE UNLOGGED TABLE t (val vector(3));
@@ -143,62 +103,10 @@ DETAIL:  Valid values are between "1" and "32768".
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
 ERROR:  value 32769 out of bounds for option "lists"
 DETAIL:  Valid values are between "1" and "32768".
-DROP TABLE t;
 SHOW ivfflat.probes;
  ivfflat.probes 
 ----------------
  1
 (1 row)
 
-SET ivfflat.probes = 0;
-ERROR:  0 is outside the valid range for parameter "ivfflat.probes" (1 .. 32768)
-SET ivfflat.probes = 32769;
-ERROR:  32769 is outside the valid range for parameter "ivfflat.probes" (1 .. 32768)
-SHOW ivfflat.iterative_scan;
- ivfflat.iterative_scan 
-------------------------
- off
-(1 row)
-
-SET ivfflat.iterative_scan = on;
-ERROR:  invalid value for parameter "ivfflat.iterative_scan": "on"
-HINT:  Available values: off, relaxed_order.
-SHOW ivfflat.max_probes;
- ivfflat.max_probes 
---------------------
- 32768
-(1 row)
-
-SET ivfflat.max_probes = 0;
-ERROR:  0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
-SET ivfflat.max_probes = 32769;
-ERROR:  32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
--- dimensions
-CREATE TABLE t (val vector(2000));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
 DROP TABLE t;
-CREATE TABLE t (val vector(2001));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-ERROR:  column cannot have more than 2000 dimensions for ivfflat index
-DROP TABLE t;
--- memory
-SET maintenance_work_mem = '1MB';
-CREATE TABLE t (val vector(2000));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-RESET maintenance_work_mem;
-SET maintenance_work_mem = '5MB';
-CREATE TABLE t (val vector(2000));
-INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-NOTICE:  ivfflat index created with little data
-DETAIL:  This will cause low recall.
-HINT:  Drop the index until the table has more data.
-DROP TABLE t;
-RESET maintenance_work_mem;

test/expected/vector_type.out

@@ -576,12 +576,6 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
  01001110101
 (1 row)
 
-SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
-   binary_quantize   
----------------------
- 1110110110011011011
-(1 row)
-
 SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
  subvector 
 -----------

test/sql/halfvec.sql

@@ -121,7 +121,6 @@ SELECT l2_normalize('[65504]'::halfvec);
 
 SELECT binary_quantize('[1,0,-1]'::halfvec);
 SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
-SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
 
 SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
 SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);

test/sql/hnsw_bit.sql

@@ -33,13 +33,3 @@ CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
 CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
 CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
 DROP TABLE t;
-
--- dimensions
-
-CREATE TABLE t (val bit(64000));
-CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
-DROP TABLE t;
-
-CREATE TABLE t (val bit(64001));
-CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
-DROP TABLE t;

test/sql/hnsw_halfvec.sql

@@ -56,13 +56,3 @@ SELECT * FROM t ORDER BY val <+> '[3,3,3]';
 SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
 
 DROP TABLE t;
-
--- dimensions
-
-CREATE TABLE t (val halfvec(4000));
-CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
-DROP TABLE t;
-
-CREATE TABLE t (val halfvec(4001));
-CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
-DROP TABLE t;

test/sql/hnsw_vector.sql

@@ -57,26 +57,6 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2
 
 DROP TABLE t;
 
--- iterative
-
-CREATE TABLE t (val vector(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING hnsw (val vector_l2_ops);
-
-SET hnsw.iterative_scan = strict_order;
-SET hnsw.ef_search = 1;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-SET hnsw.iterative_scan = relaxed_order;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-RESET hnsw.iterative_scan;
-RESET hnsw.ef_search;
-DROP TABLE t;
-
 -- unlogged
 
 CREATE UNLOGGED TABLE t (val vector(3));
@@ -95,28 +75,10 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
-DROP TABLE t;
 
 SHOW hnsw.ef_search;
+
 SET hnsw.ef_search = 0;
 SET hnsw.ef_search = 1001;
 
-SHOW hnsw.iterative_scan;
-SET hnsw.iterative_scan = on;
-
-SHOW hnsw.max_scan_tuples;
-SET hnsw.max_scan_tuples = 0;
-
-SHOW hnsw.scan_mem_multiplier;
-SET hnsw.scan_mem_multiplier = 0;
-SET hnsw.scan_mem_multiplier = 1001;
-
--- dimensions
-
-CREATE TABLE t (val vector(2000));
-CREATE INDEX ON t USING hnsw (val vector_l2_ops);
-DROP TABLE t;
-
-CREATE TABLE t (val vector(2001));
-CREATE INDEX ON t USING hnsw (val vector_l2_ops);
 DROP TABLE t;

test/sql/ivfflat_bit.sql

@@ -21,28 +21,3 @@ CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1)
 CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
 CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
 DROP TABLE t;
-
--- dimensions
-
-CREATE TABLE t (val bit(64000));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-DROP TABLE t;
-
-CREATE TABLE t (val bit(64001));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-DROP TABLE t;
-
--- memory
-
-SET maintenance_work_mem = '1MB';
-CREATE TABLE t (val bit(64000));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-DROP TABLE t;
-RESET maintenance_work_mem;
-
-SET maintenance_work_mem = '29MB';
-CREATE TABLE t (val bit(64000));
-INSERT INTO t (val) VALUES (B'0'::bit(64000));
-CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
-DROP TABLE t;
-RESET maintenance_work_mem;

test/sql/ivfflat_halfvec.sql

@@ -43,28 +43,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
 SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
 
 DROP TABLE t;
-
--- dimensions
-
-CREATE TABLE t (val halfvec(4000));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-DROP TABLE t;
-
-CREATE TABLE t (val halfvec(4001));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-DROP TABLE t;
-
--- memory
-
-SET maintenance_work_mem = '1MB';
-CREATE TABLE t (val halfvec(4000));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-DROP TABLE t;
-RESET maintenance_work_mem;
-
-SET maintenance_work_mem = '6MB';
-CREATE TABLE t (val halfvec(4000));
-INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
-CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
-DROP TABLE t;
-RESET maintenance_work_mem;

test/sql/ivfflat_vector.sql

@@ -44,28 +44,6 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2
 
 DROP TABLE t;
 
--- iterative
-
-CREATE TABLE t (val vector(3));
-INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 3);
-
-SET ivfflat.iterative_scan = relaxed_order;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-SET ivfflat.max_probes = 1;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-SET ivfflat.max_probes = 2;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
-
-RESET ivfflat.iterative_scan;
-RESET ivfflat.max_probes;
-DROP TABLE t;
-
 -- unlogged
 
 CREATE UNLOGGED TABLE t (val vector(3));
@@ -81,40 +59,7 @@ DROP TABLE t;
 CREATE TABLE t (val vector(3));
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
-DROP TABLE t;
 
 SHOW ivfflat.probes;
-SET ivfflat.probes = 0;
-SET ivfflat.probes = 32769;
 
-SHOW ivfflat.iterative_scan;
-SET ivfflat.iterative_scan = on;
-
-SHOW ivfflat.max_probes;
-SET ivfflat.max_probes = 0;
-SET ivfflat.max_probes = 32769;
-
--- dimensions
-
-CREATE TABLE t (val vector(2000));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
 DROP TABLE t;
-
-CREATE TABLE t (val vector(2001));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-DROP TABLE t;
-
--- memory
-
-SET maintenance_work_mem = '1MB';
-CREATE TABLE t (val vector(2000));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-DROP TABLE t;
-RESET maintenance_work_mem;
-
-SET maintenance_work_mem = '5MB';
-CREATE TABLE t (val vector(2000));
-INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
-CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
-DROP TABLE t;
-RESET maintenance_work_mem;

test/sql/vector_type.sql

@@ -128,7 +128,6 @@ SELECT l2_normalize('[3e38]'::vector);
 
 SELECT binary_quantize('[1,0,-1]'::vector);
 SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
-SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
 
 SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
 SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);

test/t/002_ivfflat_vacuum.pl

@@ -6,7 +6,13 @@ use Test::More;
 
 my $dim = 3;
 
-my $array_sql = join(",", ('random()') x $dim);
+my @r = ();
+for (1 .. $dim)
+{
+	my $v = int(rand(1000)) + 1;
+	push(@r, "i % $v");
+}
+my $array_sql = join(", ", @r);
 
 # Initialize node
 my $node = PostgreSQL::Test::Cluster->new('node');
@@ -17,20 +23,19 @@ $node->start;
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
 $node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
 $node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
+	"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
 );
 $node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
 
 # Get size
 my $size = $node->safe_psql("postgres", "SELECT pg_total_relation_size('tst_v_idx');");
 
-# Store values
-$node->safe_psql("postgres", "CREATE TABLE tmp AS SELECT * FROM tst;");
-
 # Delete all, vacuum, and insert same data
 $node->safe_psql("postgres", "DELETE FROM tst;");
 $node->safe_psql("postgres", "VACUUM tst;");
-$node->safe_psql("postgres", "INSERT INTO tst SELECT * FROM tmp;");
+$node->safe_psql("postgres",
+	"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
+);
 
 # Check size
 my $new_size = $node->safe_psql("postgres", "SELECT pg_total_relation_size('tst_v_idx');");

test/t/005_ivfflat_query_recall.pl

@@ -16,19 +16,29 @@ $node->safe_psql("postgres",
 	"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
 );
 
-# Add index
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
+# Check each index type
+my @operators = ("<->", "<#>", "<=>");
+my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
 
-# Test 100% recall
-for (1 .. 20)
+for my $i (0 .. $#operators)
 {
-	my $id = int(rand() * 100000);
-	my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
-	my $res = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
-	));
-	is($res, $query);
+	my $operator = $operators[$i];
+	my $opclass = $opclasses[$i];
+
+	# Add index
+	$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
+
+	# Test 100% recall
+	for (1 .. 20)
+	{
+		my $id = int(rand() * 100000);
+		my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
+		my $res = $node->safe_psql("postgres", qq(
+			SET enable_seqscan = off;
+			SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
+		));
+		is($res, $query);
+	}
 }
 
 done_testing();

test/t/041_hnsw_streaming.pl

@@ -26,26 +26,25 @@ $node->safe_psql("postgres", qq(
 
 my $count = $node->safe_psql("postgres", qq(
 	SET enable_seqscan = off;
-	SET hnsw.iterative_scan = relaxed_order;
-	SET hnsw.max_scan_tuples = 100000;
-	SET hnsw.scan_mem_multiplier = 2;
+	SET hnsw.streaming = on;
+	SET work_mem = '8MB';
 	SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
 ));
 is($count, 10);
 
 foreach ((30000, 50000, 70000))
 {
-	my $max_tuples = $_;
-	my $expected = $max_tuples / 10000;
+	my $ef_stream = $_;
+	my $expected = $ef_stream / 10000;
 	my $sum = 0;
 
 	for my $i (1 .. 20)
 	{
 		$count = $node->safe_psql("postgres", qq(
 			SET enable_seqscan = off;
-			SET hnsw.iterative_scan = relaxed_order;
-			SET hnsw.max_scan_tuples = $max_tuples;
-			SET hnsw.scan_mem_multiplier = 2;
+			SET hnsw.streaming = on;
+			SET hnsw.ef_stream = $ef_stream;
+			SET work_mem = '8MB';
 			SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst WHERE i = $i) LIMIT 11) t;
 		));
 		$sum += $count;
@@ -56,4 +55,13 @@ foreach ((30000, 50000, 70000))
 	cmp_ok($avg, '<', $expected + 2);
 }
 
+my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
+	SET enable_seqscan = off;
+	SET hnsw.streaming = on;
+	SET client_min_messages = debug1;
+	SET work_mem = '2MB';
+	SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
+));
+like($stderr, qr/hnsw index scan exceeded work_mem after \d+ tuples/);
+
 done_testing();

test/t/041_ivfflat_iterative_scan.pl

@@ -1,54 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $dim = 3;
-my $array_sql = join(",", ('random()') x $dim);
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i int4 PRIMARY KEY, v vector($dim));");
-$node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
-);
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
-
-my $count = $node->safe_psql("postgres", qq(
-	SET enable_seqscan = off;
-	SET ivfflat.probes = 10;
-	SET ivfflat.iterative_scan = relaxed_order;
-	SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
-));
-is($count, 10);
-
-foreach ((30, 50, 70))
-{
-	my $max_probes = $_;
-	my $expected = $max_probes / 10;
-	my $sum = 0;
-
-	for my $i (1 .. 20)
-	{
-		$count = $node->safe_psql("postgres", qq(
-			SET enable_seqscan = off;
-			SET ivfflat.probes = 10;
-			SET ivfflat.iterative_scan = relaxed_order;
-			SET ivfflat.max_probes = $max_probes;
-			SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst WHERE i = $i) LIMIT 11) t;
-		));
-		$sum += $count;
-	}
-
-	my $avg = $sum / 20;
-	cmp_ok($avg, '>', $expected - 2);
-	cmp_ok($avg, '<', $expected + 2);
-}
-
-done_testing();

test/t/042_hnsw_streaming_recall.pl

@@ -10,18 +10,18 @@ my @expected;
 my $limit = 20;
 my $dim = 3;
 my $array_sql = join(",", ('random()') x $dim);
-my @cs = (50, 500);
+my @cs = (100, 1000);
 
 sub test_recall
 {
-	my ($c, $ef_search, $min, $operator, $mode) = @_;
+	my ($c, $ef_search, $min, $operator) = @_;
 	my $correct = 0;
 	my $total = 0;
 
 	my $explain = $node->safe_psql("postgres", qq(
 		SET enable_seqscan = off;
 		SET hnsw.ef_search = $ef_search;
-		SET hnsw.iterative_scan = $mode;
+		SET hnsw.streaming = on;
 		EXPLAIN ANALYZE SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[0]' LIMIT $limit;
 	));
 	like($explain, qr/Index Scan using idx on tst/);
@@ -31,7 +31,7 @@ sub test_recall
 		my $actual = $node->safe_psql("postgres", qq(
 			SET enable_seqscan = off;
 			SET hnsw.ef_search = $ef_search;
-			SET hnsw.iterative_scan = $mode;
+			SET hnsw.streaming = on;
 			SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[$i]' LIMIT $limit;
 		));
 		my @actual_ids = split("\n", $actual);
@@ -50,7 +50,7 @@ sub test_recall
 		$total += $limit;
 	}
 
-	cmp_ok($correct / $total, ">=", $min, "$operator $mode $c");
+	cmp_ok($correct / $total, ">=", $min, $operator);
 }
 
 # Initialize node
@@ -62,7 +62,7 @@ $node->start;
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
 $node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
 $node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 50000) i;"
+	"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
 );
 
 # Generate queries
@@ -108,8 +108,21 @@ for my $i (0 .. $#operators)
 			push(@expected, $res);
 		}
 
-		test_recall($c, 40, 0.99, $operator, "strict_order");
-		test_recall($c, 40, 0.99, $operator, "relaxed_order");
+		if ($c == 100)
+		{
+			test_recall($c, 40, 0.99, $operator);
+		}
+		else
+		{
+			if ($operator eq "<->")
+			{
+				test_recall($c, 40, 0.99, $operator);
+			}
+			else
+			{
+				test_recall($c, 40, 0.99, $operator);
+			}
+		}
 	}
 
 	$node->safe_psql("postgres", "DROP INDEX idx;");

test/t/042_ivfflat_iterative_scan_recall.pl

@@ -1,125 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $node;
-my @queries = ();
-my @expected;
-my $limit = 20;
-my @cs = (100, 1000);
-
-sub test_recall
-{
-	my ($c, $probes, $min, $operator) = @_;
-	my $correct = 0;
-	my $total = 0;
-
-	my $explain = $node->safe_psql("postgres", qq(
-		SET enable_seqscan = off;
-		SET ivfflat.probes = $probes;
-		SET ivfflat.iterative_scan = relaxed_order;
-		EXPLAIN ANALYZE SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[0]' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan using idx on tst/);
-
-	for my $i (0 .. $#queries)
-	{
-		my $actual = $node->safe_psql("postgres", qq(
-			SET enable_seqscan = off;
-			SET ivfflat.probes = $probes;
-			SET ivfflat.iterative_scan = relaxed_order;
-			SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[$i]' LIMIT $limit;
-		));
-		my @actual_ids = split("\n", $actual);
-
-		my @expected_ids = split("\n", $expected[$i]);
-		my %expected_set = map { $_ => 1 } @expected_ids;
-
-		foreach (@actual_ids)
-		{
-			if (exists($expected_set{$_}))
-			{
-				$correct++;
-			}
-		}
-
-		$total += $limit;
-	}
-
-	cmp_ok($correct / $total, ">=", $min, "$operator $c");
-}
-
-# Initialize node
-$node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector(3));");
-$node->safe_psql("postgres",
-	"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
-);
-
-# Generate queries
-for (1 .. 20)
-{
-	my $r1 = rand();
-	my $r2 = rand();
-	my $r3 = rand();
-	push(@queries, "[$r1,$r2,$r3]");
-}
-
-# Check each index type
-my @operators = ("<->", "<=>");
-my @opclasses = ("vector_l2_ops", "vector_cosine_ops");
-
-for my $i (0 .. $#operators)
-{
-	my $operator = $operators[$i];
-	my $opclass = $opclasses[$i];
-
-	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v $opclass);");
-
-	foreach (@cs)
-	{
-		my $c = $_;
-
-		# Get exact results
-		@expected = ();
-		foreach (@queries)
-		{
-			my $res = $node->safe_psql("postgres", qq(
-				SET enable_indexscan = off;
-				WITH top AS (
-					SELECT v $operator '$_' AS distance FROM tst WHERE i % $c = 0 ORDER BY distance LIMIT $limit
-				)
-				SELECT i FROM tst WHERE (v $operator '$_') <= (SELECT MAX(distance) FROM top)
-			));
-			push(@expected, $res);
-		}
-
-		if ($c == 100)
-		{
-			test_recall($c, 1, 0.57, $operator);
-			test_recall($c, 10, 0.98, $operator);
-		}
-		else
-		{
-			if ($operator eq "<->")
-			{
-				test_recall($c, 1, 0.80, $operator);
-			}
-			else
-			{
-				test_recall($c, 1, 0.88, $operator);
-			}
-		}
-	}
-
-	$node->safe_psql("postgres", "DROP INDEX idx;");
-}
-
-done_testing();

test/t/045_hnsw_low_memory_build.pl

@@ -1,29 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
-$node->safe_psql("postgres",
-	"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 1000) i;"
-);
-
-my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
-	SET client_min_messages = DEBUG;
-	SET maintenance_work_mem = '3073kB';
-	ALTER TABLE tst SET (parallel_workers = 1);
-	CREATE INDEX ON tst USING hnsw (v vector_l2_ops);
-));
-is($ret, 0, $stderr);
-like($stderr, qr/using \d+ parallel workers/);
-like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem after 0 tuples/);
-
-done_testing();

test/t/046_hnsw_vacuum_scan.pl

@@ -1,38 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $dim = 3;
-my $array_sql = join(",", ('random()') x $dim);
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table and index
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
-$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
-$node->safe_psql("postgres",
-    "INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
-);
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
-$node->safe_psql("postgres", "DELETE FROM tst");
-
-# Test HNSW_SCAN_LOCK at the beginning of MarkDeleted is effective
-$node->pgbench(
-    "--no-vacuum --client=5 --transactions=1000",
-    0,
-    [qr{actually processed}],
-    [qr{^$}],
-    "concurrent SELECTs and VACUUM",
-    {
-        "046_hnsw_vacuum_scan_select\@1000" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
-        "046_hnsw_vacuum_scan_vacuum\@1" => "VACUUM tst;"
-    }
-);
-
-done_testing();

test/t/047_hnsw_vacuum_insert.pl

@@ -1,39 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $dim = 3;
-my $array_sql = join(",", ('random()') x $dim);
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table and index
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
-$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
-$node->safe_psql("postgres",
-    "INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
-);
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
-
-# Test no "hnsw graph not repaired" errors
-$node->pgbench(
-    "--no-vacuum --client=5 --transactions=1000",
-    0,
-    [qr{actually processed}],
-    [qr{^$}],
-    "concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
-    {
-        "047_hnsw_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
-        "047_hnsw_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
-        "047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
-        "047_hnsw_vacuum_insert_vacuum\@1" => "VACUUM tst;"
-    }
-);
-
-done_testing();

test/t/048_ivfflat_vacuum_insert.pl

@@ -1,39 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my $dim = 3;
-my $array_sql = join(",", ('random()') x $dim);
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-# Create table and index
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
-$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
-$node->safe_psql("postgres",
-    "INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
-);
-$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 10);");
-
-# Test no errors
-$node->pgbench(
-    "--no-vacuum --client=5 --transactions=1500",
-    0,
-    [qr{actually processed}],
-    [qr{^$}],
-    "concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
-    {
-        "048_ivfflat_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
-        "048_ivfflat_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
-        "048_ivfflat_vacuum_insert_select\@500" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
-        "048_ivfflat_vacuum_insert_vacuum\@1" => "VACUUM tst;"
-    }
-);
-
-done_testing();

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat and hnsw access methods'
-default_version = '0.8.5'
+default_version = '0.7.4'
 module_pathname = '$libdir/vector'
 relocatable = true