Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
42c930231a Test with ARM on CI 2025-01-19 01:29:28 -08:00
43 changed files with 262 additions and 927 deletions

8
.dockerignore Normal file
View File

@@ -0,0 +1,8 @@
/.git/
/dist/
/results/
/tmp_check/
/sql/vector--?.?.?.sql
regression.*
*.o
*.so

View File

@@ -8,29 +8,19 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 19
os: ubuntu-24.04
- postgres: 18
os: ubuntu-24.04
- postgres: 17
os: ubuntu-24.04
- postgres: 16
os: ubuntu-24.04-arm
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-22.04-arm
- postgres: 13
os: ubuntu-22.04
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
@@ -41,103 +31,3 @@ jobs:
sudo apt-get update
sudo apt-get install libipc-run-perl
- run: make prove_installcheck
mac:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 18
os: macos-26
- postgres: 14
os: macos-15-intel
steps:
- uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
# Homebrew Postgres does not enable TAP tests, so need to download
- run: |
brew install cpanm
cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
env:
TAG: ${{ matrix.postgres == 18 && 'REL_18_2' || 'REL_14_21' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make
env:
LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'mac') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 17
os: windows-2025
- postgres: 14
os: windows-2022
steps:
- uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
- if: ${{ failure() }}
run: cat regression.diffs
i386:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
container:
image: debian:12
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- if: ${{ failure() }}
run: cat pgvector/regression.diffs
valgrind:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 18
check-ub: yes
- run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install
- run: make installcheck

View File

@@ -1,14 +1,3 @@
## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)
- Improved `install` target on Windows
- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
## 0.8.1 (2025-09-04)
- Added support for Postgres 18 rc1
- Improved performance of `binary_quantize` function
## 0.8.0 (2024-10-30)
- Added support for iterative index scans

View File

@@ -1,11 +1,8 @@
# syntax=docker/dockerfile:1
ARG PG_MAJOR=17
ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
FROM postgres:$PG_MAJOR
ARG PG_MAJOR
ADD https://github.com/pgvector/pgvector.git#v0.8.2 /tmp/pgvector
COPY . /tmp/pgvector
RUN apt-get update && \
apt-mark hold locales && \

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.2",
"version": "0.8.0",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.8.2",
"version": "0.8.0",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.2
EXTVERSION = 0.8.0
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)
@@ -27,11 +27,6 @@ ifneq ($(filter ppc64%, $(shell uname -m)), )
OPTFLAGS =
endif
# RISC-V64 doesn't support -march=native
ifeq ($(shell uname -m), riscv64)
OPTFLAGS =
endif
# For auto-vectorization:
# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
# - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -81,9 +76,4 @@ docker:
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm .
.PHONY: docker-release-trixie
docker-release-trixie:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie .
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.2
EXTVERSION = 0.8.0
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
@@ -31,9 +31,6 @@ LIBDIR = $(PGROOT)\lib
PKGLIBDIR = $(PGROOT)\lib
SHAREDIR = $(PGROOT)\share
# Use $(PGROOT)\bin\pg_regress for Postgres < 17
PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
@@ -57,11 +54,11 @@ install: all
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
installcheck:
"$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
uninstall:
del /f "$(PKGLIBDIR)\$(SHLIB)"

206
README.md
View File

@@ -21,7 +21,7 @@ Compile and install the extension (supports Postgres 13+)
```sh
cd /tmp
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -29,21 +29,31 @@ make install # may need sudo
See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), [APK](#apk), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
### Windows
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build:
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\18"
call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
```
Note: The exact path will vary depending on your Visual Studio version and edition
Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP%
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
```
Note: Postgres 17 is not supported with MSVC yet due to an [upstream issue](https://www.postgresql.org/message-id/flat/CAOdR5yF0krWrxycA04rgUKCgKugRvGWzzGLAhDZ9bzNv8g0Lag%40mail.gmail.com)
See the [installation notes](#installation-notes---windows) if you run into issues
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -74,7 +84,7 @@ Get the nearest neighbors by L2 distance
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`)
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
@@ -138,9 +148,9 @@ Supported distance functions are:
- `<->` - L2 distance
- `<#>` - (negative) inner product
- `<=>` - cosine distance
- `<+>` - L1 distance
- `<~>` - Hamming distance (binary vectors)
- `<%>` - Jaccard distance (binary vectors)
- `<+>` - L1 distance (added in 0.7.0)
- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
Get the nearest neighbors to a row
@@ -227,19 +237,19 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
```
L1 distance
L1 distance - added in 0.7.0
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
```
Hamming distance
Hamming distance - added in 0.7.0
```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
```
Jaccard distance
Jaccard distance - added in 0.7.0
```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
@@ -248,9 +258,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions
- `bit` - up to 64,000 dimensions
- `sparsevec` - up to 1,000 non-zero elements
- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions (added in 0.7.0)
- `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0)
### Index Options
@@ -304,15 +314,13 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t
Like other index types, its faster to create an index after loading your initial data
You can also speed up index creation by increasing the number of parallel workers (2 by default)
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql
SET max_parallel_maintenance_workers = 7; -- plus leader
```
For a large number of workers, you may need to increase `max_parallel_workers` (8 by default)
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
### Indexing Progress
@@ -359,7 +367,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
```
Hamming distance
Hamming distance - added in 0.7.0
```sql
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
@@ -368,8 +376,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions
- `bit` - up to 64,000 dimensions
- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions (added in 0.7.0)
### Query Options
@@ -469,6 +477,8 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
## Iterative Index Scans
*Added in 0.8.0*
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
Iterative scans can use strict or relaxed ordering.
@@ -492,11 +502,9 @@ With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.o
```sql
WITH relaxed_results AS MATERIALIZED (
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
) SELECT * FROM relaxed_results ORDER BY distance + 0;
) SELECT * FROM relaxed_results ORDER BY distance;
```
Note: `+ 0` is needed for Postgres 17+
For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
```sql
@@ -541,6 +549,8 @@ Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used
## Half-Precision Vectors
*Added in 0.7.0*
Use the `halfvec` type to store half-precision vectors
```sql
@@ -549,6 +559,8 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
## Half-Precision Indexing
*Added in 0.7.0*
Index vectors at half precision for smaller indexes
```sql
@@ -570,16 +582,24 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111');
```
Get the nearest neighbors by Hamming distance
Get the nearest neighbors by Hamming distance (added in 0.7.0)
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Or (before 0.7.0)
```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
```
Also supports Jaccard distance (`<%>`)
## Binary Quantization
*Added in 0.7.0*
Use expression indexing for binary quantization
```sql
@@ -602,6 +622,8 @@ SELECT * FROM (
## Sparse Vectors
*Added in 0.7.0*
Use the `sparsevec` type to store sparse vectors
```sql
@@ -635,6 +657,8 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
## Indexing Subvectors
*Added in 0.7.0*
Use expression indexing to index subvectors
```sql
@@ -695,10 +719,10 @@ CREATE INDEX CONCURRENTLY ...
### Querying
Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance.
Use `EXPLAIN ANALYZE` to debug performance.
```sql
EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
#### Exact Search
@@ -748,6 +772,8 @@ SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_t
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
Monitor recall by comparing results from approximate search with exact search.
```sql
@@ -771,12 +797,9 @@ Use pgvector from any language with a Postgres client. You can even generate and
Language | Libraries / Examples
--- | ---
Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada)
Algol | [pgvector-algol](https://github.com/pgvector/pgvector-algol)
C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
COBOL | [pgvector-cobol](https://github.com/pgvector/pgvector-cobol)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
@@ -788,23 +811,19 @@ Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl)
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml)
Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal)
Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Prolog | [pgvector-prolog](https://github.com/pgvector/pgvector-prolog)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Racket | [pgvector-racket](https://github.com/pgvector/pgvector-racket)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
Tcl | [pgvector-tcl](https://github.com/pgvector/pgvector-tcl)
Zig | [pgvector-zig](https://github.com/pgvector/pgvector-zig)
## Frequently Asked Questions
@@ -819,11 +838,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions?
You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
#### Can I store vectors with different dimensions in the same column?
You can use `vector` as the type (instead of `vector(n)`).
You can use `vector` as the type (instead of `vector(3)`).
```sql
CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
@@ -1071,7 +1090,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
```sh
export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config
export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
```
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1082,11 +1101,11 @@ sudo --preserve-env=PG_CONFIG make install
A few common paths on Mac are:
- EDB installer - `/Library/PostgreSQL/18/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config`
- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
Note: Replace `18` with your Postgres server version
Note: Replace `17` with your Postgres server version
### Missing Header
@@ -1095,20 +1114,14 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
For Ubuntu and Debian, use:
```sh
sudo apt install postgresql-server-dev-18
sudo apt install postgresql-server-dev-17
```
Note: Replace `18` with your Postgres server version
Note: Replace `17` with your Postgres server version
### Missing SDK
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists.
```sh
pg_config --cppflags
```
Reinstall Postgres to fix this.
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
### Portability
@@ -1126,14 +1139,6 @@ make OPTFLAGS=""
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
### Mismatched Architecture
If compilation fails with `error C2196: case value '4' already used`, make sure youre using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
### Missing Symbol
If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+.
### Permissions
If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
@@ -1145,38 +1150,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
```sh
docker pull pgvector/pgvector:pg18-trixie
docker pull pgvector/pgvector:pg17
```
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way).
Supported tags are:
- `pg18-trixie`, `0.8.2-pg18-trixie`
- `pg18-bookworm`, `0.8.2-pg18-bookworm`, `pg18`, `0.8.2-pg18`
- `pg17-trixie`, `0.8.2-pg17-trixie`
- `pg17-bookworm`, `0.8.2-pg17-bookworm`, `pg17`, `0.8.2-pg17`
- `pg16-trixie`, `0.8.2-pg16-trixie`
- `pg16-bookworm`, `0.8.2-pg16-bookworm`, `pg16`, `0.8.2-pg16`
- `pg15-trixie`, `0.8.2-pg15-trixie`
- `pg15-bookworm`, `0.8.2-pg15-bookworm`, `pg15`, `0.8.2-pg15`
- `pg14-trixie`, `0.8.2-pg14-trixie`
- `pg14-bookworm`, `0.8.2-pg14-bookworm`, `pg14`, `0.8.2-pg14`
- `pg13-trixie`, `0.8.2-pg13-trixie`
- `pg13-bookworm`, `0.8.2-pg13-bookworm`, `pg13`, `0.8.2-pg13`
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
You can also build the image manually:
```sh
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
```
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
```
### Homebrew
@@ -1187,7 +1171,7 @@ With Homebrew Postgres, you can use:
brew install pgvector
```
Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas
Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
### PGXN
@@ -1202,29 +1186,29 @@ pgxn install vector
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
```sh
sudo apt install postgresql-18-pgvector
sudo apt install postgresql-17-pgvector
```
Note: Replace `18` with your Postgres server version
Note: Replace `17` with your Postgres server version
### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
```sh
sudo yum install pgvector_18
sudo yum install pgvector_17
# or
sudo dnf install pgvector_18
sudo dnf install pgvector_17
```
Note: Replace `18` with your Postgres server version
Note: Replace `17` with your Postgres server version
### pkg
Install the FreeBSD package with:
```sh
pkg install postgresql17-pgvector
pkg install postgresql16-pgvector
```
or the port with:
@@ -1234,14 +1218,6 @@ cd /usr/ports/databases/pgvector
make install
```
### APK
Install the Alpine package with:
```sh
apk add postgresql-pgvector
```
### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
@@ -1274,6 +1250,36 @@ You can check the version in the current database with:
SELECT extversion FROM pg_extension WHERE extname = 'vector';
```
## Upgrade Notes
### 0.6.0
#### Postgres 12
If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
```sql
ALTER TYPE vector SET (STORAGE = external);
```
Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
#### Docker
The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
```sh
docker pull pgvector/pgvector:pg16
# or
docker pull pgvector/pgvector:0.6.0-pg16
```
Also, if youve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
```
## Thanks
Thanks to:

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.2'" to load this file. \quit

View File

@@ -169,7 +169,7 @@ BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char
#endif
TARGET_XSAVE static bool
SupportsAvx512Popcount(void)
SupportsAvx512Popcount()
{
unsigned int exx[4] = {0, 0, 0, 0};

View File

@@ -2,7 +2,6 @@
#include "bitutils.h"
#include "bitvec.h"
#include "fmgr.h"
#include "utils/varbit.h"
#include "vector.h"

View File

@@ -1,7 +1,5 @@
#include "postgres.h"
#include <math.h>
#include "halfutils.h"
#include "halfvec.h"

View File

@@ -13,20 +13,12 @@
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/varbit.h"
#include "utils/numeric.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -137,9 +129,9 @@ InitHalfVector(int dim)
return result;
}
#if PG_VERSION_NUM >= 170000
#define halfvec_isspace(ch) scanner_isspace(ch)
#else
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
halfvec_isspace(char ch)
{
@@ -152,7 +144,6 @@ halfvec_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -907,21 +898,8 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
half *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
/* Auto-vectorized on aarch64 */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (HalfToFloat4(ax[i + j]) > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
for (int i = 0; i < a->dim; i++)
rx[i / 8] |= (HalfToFloat4(ax[i]) > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);

View File

@@ -1,25 +1,18 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "storage/lwlock.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -59,20 +52,12 @@ HnswInitLockTranche(void)
sizeof(int) * 1,
&found);
if (!found)
{
#if PG_VERSION_NUM >= 190000
tranche_ids[0] = LWLockNewTrancheId("HnswBuild");
#else
tranche_ids[0] = LWLockNewTrancheId();
#endif
}
hnsw_lock_tranche_id = tranche_ids[0];
LWLockRelease(AddinShmemInitLock);
#if PG_VERSION_NUM < 190000
/* Per-backend registration of the tranche ID */
LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
#endif
}
/*
@@ -145,7 +130,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Relation index;
/* Never use index without order */
if (path->indexorderbys == NIL)
if (path->indexorderbys == NULL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
@@ -267,64 +252,6 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
Datum
hnswhandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 3,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = hnswbuild,
.ambuildempty = hnswbuildempty,
.aminsert = hnswinsert,
.aminsertcleanup = NULL,
.ambulkdelete = hnswbulkdelete,
.amvacuumcleanup = hnswvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = hnswcostestimate,
.amgettreeheight = NULL,
.amoptions = hnswoptions,
.amproperty = NULL,
.ambuildphasename = hnswbuildphasename,
.amvalidate = hnswvalidate,
.amadjustmembers = NULL,
.ambeginscan = hnswbeginscan,
.amrescan = hnswrescan,
.amgettuple = hnswgettuple,
.amgetbitmap = NULL,
.amendscan = hnswendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
@@ -332,11 +259,6 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -369,9 +291,6 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL;
amroutine->amcostestimate = hnswcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = hnswoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = hnswbuildphasename;
@@ -392,11 +311,5 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -3,25 +3,15 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "lib/pairingheap.h"
#include "nodes/execnodes.h"
#include "port.h" /* for random() */
#include "storage/bufpage.h"
#include "storage/condition_variable.h"
#include "storage/lwlock.h"
#include "storage/s_lock.h"
#include "utils/relptr.h"
#include "utils/sampling.h"
#include "vector.h"
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define HNSW_MAX_DIM 2000
#define HNSW_MAX_NNZ 1000
@@ -372,13 +362,6 @@ typedef union
ItemPointerData indextid;
} HnswUnvisited;
typedef struct HnswReadStreamData
{
HnswUnvisited *unvisited;
int unvisitedLength;
int visited;
} HnswReadStreamData;
typedef struct HnswScanOpaqueData
{
const HnswTypeInfo *typeInfo;
@@ -434,14 +417,14 @@ bool HnswCheckNorm(HnswSupport * support, Datum value);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page);
void HnswInit(void);
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples, bool maintenance);
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing, bool maintenance);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);

View File

@@ -36,14 +36,11 @@
*/
#include "postgres.h"
#include <limits.h>
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/xact.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
@@ -51,19 +48,11 @@
#include "commands/progress.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -80,8 +69,6 @@
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
#define HNSW_MAX_GRAPH_MEMORY (SIZE_MAX / 2)
/*
* Create the metapage
*/
@@ -411,7 +398,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
* Update graph in memory
*/
static void
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate)
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
{
HnswGraph *graph = buildstate->graph;
char *base = buildstate->hnswarea;
@@ -470,10 +457,10 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
}
/* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false, true);
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
/* Update graph in memory */
UpdateGraphInMemory(support, element, m, entryPoint, buildstate);
UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate);
/* Release entry lock */
LWLockRelease(entryLock);
@@ -494,7 +481,6 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
LWLock *flushLock = &graph->flushLock;
char *base = buildstate->hnswarea;
Datum value;
Size memoryMargin;
/* Form index value */
if (!HnswFormIndexValue(&value, values, isnull, buildstate->typeInfo, support))
@@ -503,9 +489,6 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Get datum size */
valueSize = VARSIZE_ANY(DatumGetPointer(value));
/* In a parallel build, add a margin so allocations never fail */
memoryMargin = base == NULL ? 0 : 1024 * 1024;
/* Ensure graph not flushed when inserting */
LWLockAcquire(flushLock, LW_SHARED);
@@ -527,7 +510,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Check that we have enough memory available for the new element now that
* we have the allocator lock, and flush pages if needed.
*/
if (graph->memoryUsed + memoryMargin >= graph->memoryTotal)
if (graph->memoryUsed >= graph->memoryTotal)
{
LWLockRelease(&graph->allocatorLock);
@@ -562,7 +545,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Copy the datum */
memcpy(valuePtr, DatumGetPointer(value), valueSize);
HnswPtrStore(base, element->value, (char *) valuePtr);
HnswPtrStore(base, element->value, valuePtr);
/* Create a lock for the element */
LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -620,7 +603,7 @@ InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
HnswPtrStore(base, graph->head, (HnswElement) NULL);
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
graph->memoryUsed = 0;
graph->memoryTotal = Min(memoryTotal, HNSW_MAX_GRAPH_MEMORY);
graph->memoryTotal = memoryTotal;
graph->flushed = false;
graph->indtuples = 0;
SpinLockInit(&graph->lock);
@@ -661,17 +644,9 @@ static void *
HnswSharedMemoryAlloc(Size size, void *state)
{
HnswBuildState *buildstate = (HnswBuildState *) state;
Size alignedSize = MAXALIGN(size);
void *chunk;
void *chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
if (alignedSize > 1024 * 1024)
elog(ERROR, "hnsw allocation too large");
if (buildstate->graph->memoryUsed + alignedSize > buildstate->graph->memoryTotal)
elog(ERROR, "hnsw allocator out of memory");
chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
buildstate->graph->memoryUsed += alignedSize;
buildstate->graph->memoryUsed += MAXALIGN(size);
return chunk;
}
@@ -803,11 +778,7 @@ HnswParallelScanAndInsert(Relation heapRel, Relation indexRel, HnswShared * hnsw
buildstate.hnswarea = hnswarea;
InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate);
scan = table_beginscan_parallel(heapRel,
ParallelTableScanFromHnswShared(hnswshared)
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
ParallelTableScanFromHnswShared(hnswshared));
reltuples = table_index_build_scan(heapRel, indexRel, indexInfo,
true, progress, BuildCallback,
(void *) &buildstate, scan);
@@ -961,8 +932,6 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
if (esthnswarea > estother)
esthnswarea -= estother;
esthnswarea = Min(esthnswarea, HNSW_MAX_GRAPH_MEMORY);
shm_toc_estimate_chunk(&pcxt->estimator, esthnswarea);
shm_toc_estimate_keys(&pcxt->estimator, 2);
@@ -1005,7 +974,8 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
snapshot);
hnswarea = (char *) shm_toc_allocate(pcxt->toc, esthnswarea);
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea);
/* Report less than allocated so never fails */
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
/*
* Avoid base address for relptr for Postgres < 14.5
@@ -1084,7 +1054,7 @@ ComputeParallelWorkers(Relation heap, Relation index)
* Build graph
*/
static void
BuildGraph(HnswBuildState * buildstate)
BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
{
int parallel_workers = 0;
@@ -1132,7 +1102,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
InitBuildState(buildstate, heap, index, indexInfo, forkNum);
BuildGraph(buildstate);
BuildGraph(buildstate, forkNum);
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);

View File

@@ -1,19 +1,13 @@
#include "postgres.h"
#include "access/genam.h"
#include <math.h>
#include "access/generic_xlog.h"
#include "hnsw.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "storage/lwlock.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Get the insert page
@@ -666,7 +660,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
* Update graph on disk
*/
static void
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building)
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
{
BlockNumber newInsertPage = InvalidBlockNumber;
@@ -714,7 +708,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
/* Create an element */
element = HnswInitElement(base, heaptid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
HnswPtrStore(base, element->value, DatumGetPointer(value));
/* Prevent concurrent inserts when likely updating entry point */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -731,10 +725,10 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
}
/* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false, building);
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
/* Update graph on disk */
UpdateGraphOnDisk(index, support, element, m, entryPoint, building);
UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building);
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);

View File

@@ -1,23 +1,12 @@
#include "postgres.h"
#include <limits.h>
#include "access/genam.h"
#include "access/relscan.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/float.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Algorithm 5 from paper
@@ -48,11 +37,11 @@ GetScanItems(IndexScanDesc scan, Datum value)
for (int lc = entryPoint->level; lc >= 1; lc--)
{
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL, false);
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL);
ep = w;
}
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples, false);
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples);
}
/*
@@ -83,7 +72,7 @@ ResumeScanItems(IndexScanDesc scan)
ep = lappend(ep, sc);
}
return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples, false);
return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
}
/*
@@ -153,7 +142,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
/* Calculate max memory */
/* Add 256 extra bytes to fill last block when close */
maxMemory = (double) work_mem * hnsw_scan_mem_multiplier * 1024.0 + 256;
so->maxMemory = Min(maxMemory, (double) (SIZE_MAX / 2));
so->maxMemory = Min(maxMemory, (double) SIZE_MAX);
scan->opaque = so;
@@ -204,10 +193,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)

View File

@@ -2,28 +2,18 @@
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "catalog/pg_type.h"
#include "catalog/pg_type_d.h"
#include "common/hashfn.h"
#include "fmgr.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "nodes/pg_list.h"
#include "port/atomics.h"
#include "sparsevec.h"
#include "storage/bufmgr.h"
#include "utils/datum.h"
#include "utils/memdebug.h"
#include "utils/rel.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 190000
#include "storage/read_stream.h"
#endif
#if PG_VERSION_NUM < 170000
static inline uint64
@@ -266,7 +256,7 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
HnswInitNeighbors(base, element, m, allocator);
HnswPtrStore(base, element->value, (char *) NULL);
HnswPtrStore(base, element->value, (Pointer) NULL);
return element;
}
@@ -292,7 +282,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
element->blkno = blkno;
element->offno = offno;
HnswPtrStore(base, element->neighbors, (HnswNeighborArrayPtr *) NULL);
HnswPtrStore(base, element->value, (char *) NULL);
HnswPtrStore(base, element->value, (Pointer) NULL);
return element;
}
@@ -518,7 +508,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
char *base = NULL;
Datum value = datumCopy(PointerGetDatum(&etup->data), false, -1);
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
HnswPtrStore(base, element->value, DatumGetPointer(value));
}
}
@@ -535,12 +525,14 @@ HnswGetDistance(Datum a, Datum b, HnswSupport * support)
* Load an element and optionally get its distance from q
*/
static void
HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance, HnswElement * element)
HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance, HnswElement * element)
{
Buffer buf;
Page page;
HnswElementTuple etup;
/* Read vector */
buf = ReadBuffer(index, blkno);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
@@ -561,7 +553,7 @@ HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery
if (distance == NULL || maxDistance == NULL || *distance < *maxDistance)
{
if (*element == NULL)
*element = HnswInitElementFromBlock(BufferGetBlockNumber(buf), offno);
*element = HnswInitElementFromBlock(blkno, offno);
HnswLoadElementFromTuple(*element, etup, true, loadVec);
}
@@ -575,9 +567,7 @@ HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery
void
HnswLoadElement(HnswElement element, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance)
{
Buffer buf = ReadBuffer(index, element->blkno);
HnswLoadElementImpl(buf, element->offno, distance, q, index, support, loadVec, maxDistance, &element);
HnswLoadElementImpl(element->blkno, element->offno, distance, q, index, support, loadVec, maxDistance, &element);
}
/*
@@ -817,31 +807,11 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
}
}
#if PG_VERSION_NUM >= 190000
/*
* Get next block number for read stream
*/
static BlockNumber
HnswReadStreamNextBlock(ReadStream *stream, void *callback_private_data, void *per_buffer_data)
{
HnswReadStreamData *streamData = callback_private_data;
OffsetNumber *offno = per_buffer_data;
HnswUnvisited *uv;
if (streamData->visited == streamData->unvisitedLength)
return InvalidBlockNumber;
uv = &streamData->unvisited[streamData->visited++];
*offno = ItemPointerGetOffsetNumber(&uv->indextid);
return ItemPointerGetBlockNumber(&uv->indextid);
}
#endif
/*
* Algorithm 2 from paper
*/
List *
HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples, bool maintenance)
HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples)
{
List *w = NIL;
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
@@ -856,21 +826,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
int unvisitedLength;
bool inMemory = index == NULL;
#if PG_VERSION_NUM >= 190000
HnswReadStreamData streamData;
ReadStream *stream = NULL;
if (!inMemory)
{
int flags = READ_STREAM_USE_BATCHING;
if (maintenance)
flags |= READ_STREAM_MAINTENANCE;
stream = read_stream_begin_relation(flags, NULL, index, MAIN_FORKNUM, HnswReadStreamNextBlock, &streamData, sizeof(OffsetNumber));
}
#endif
if (v == NULL)
{
v = &vh;
@@ -933,23 +888,13 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
if (inMemory)
HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize);
else
{
HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
#if PG_VERSION_NUM >= 190000
read_stream_resume(stream);
streamData.unvisited = unvisited;
streamData.unvisitedLength = unvisitedLength;
streamData.visited = 0;
#endif
}
/* OK to count elements instead of tuples */
if (tuples != NULL)
(*tuples) += unvisitedLength;
for (int i = 0;; i++)
for (int i = 0; i < unvisitedLength; i++)
{
HnswElement eElement;
HnswSearchCandidate *e;
@@ -960,46 +905,24 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
if (inMemory)
{
if (i == unvisitedLength)
break;
eElement = unvisited[i].element;
eDistance = GetElementDistance(base, eElement, q, support);
}
else
{
Buffer buf;
OffsetNumber offno;
#if PG_VERSION_NUM >= 190000
void *offnoPtr;
buf = read_stream_next_buffer(stream, &offnoPtr);
if (!BufferIsValid(buf))
break;
offno = *((OffsetNumber *) offnoPtr);
#else
ItemPointer indextid;
if (i == unvisitedLength)
break;
indextid = &unvisited[i].indextid;
buf = ReadBuffer(index, ItemPointerGetBlockNumber(indextid));
offno = ItemPointerGetOffsetNumber(indextid);
#endif
ItemPointer indextid = &unvisited[i].indextid;
BlockNumber blkno = ItemPointerGetBlockNumber(indextid);
OffsetNumber offno = ItemPointerGetOffsetNumber(indextid);
/* Avoid any allocations if not adding */
eElement = NULL;
HnswLoadElementImpl(buf, offno, &eDistance, q, index, support, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
HnswLoadElementImpl(blkno, offno, &eDistance, q, index, support, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
if (eElement == NULL)
continue;
}
if (!(eDistance < f->distance || alwaysAdd))
if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
{
if (discarded != NULL)
{
@@ -1049,11 +972,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
w = lappend(w, sc);
}
#if PG_VERSION_NUM >= 190000
if (!inMemory)
read_stream_end(stream);
#endif
return w;
}
@@ -1349,7 +1267,7 @@ PrecomputeHash(char *base, HnswElement element)
* Algorithm 1 from paper
*/
void
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing, bool maintenance)
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing)
{
List *ep;
List *w;
@@ -1376,7 +1294,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
/* 1st phase: greedy search to insert level */
for (int lc = entryLevel; lc >= level + 1; lc--)
{
w = HnswSearchLayer(base, &q, ep, 1, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL, maintenance);
w = HnswSearchLayer(base, &q, ep, 1, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL);
ep = w;
}
@@ -1395,7 +1313,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
List *lw = NIL;
ListCell *lc2;
w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL, maintenance);
w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL);
/* Convert search candidates to candidates */
foreach(lc2, w)

View File

@@ -1,22 +1,13 @@
#include "postgres.h"
#include "access/genam.h"
#include <math.h>
#include "access/generic_xlog.h"
#include "commands/vacuum.h"
#include "hnsw.h"
#include "nodes/pg_list.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/*
* Check if deleted list contains an index TID
@@ -179,12 +170,7 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */
if (!needsUpdated)
{
/* Keep clang-tidy happy */
Assert(ntup->count > 0);
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
}
UnlockReleaseBuffer(buf);
@@ -218,7 +204,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
element->heaptidsLength = 0;
/* Find neighbors for element, skipping itself */
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true, true);
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true);
/* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
@@ -534,9 +520,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Overwrite element */
/* Use memset instead of MemSet to keep clang-tidy happy */
etup->deleted = 1;
memset(&etup->data, 0, VARSIZE_ANY(&etup->data));
MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
/* Overwrite neighbors */
for (int i = 0; i < ntup->count; i++)

View File

@@ -2,37 +2,23 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/parallel.h"
#include "access/xact.h"
#include "access/xloginsert.h"
#include "bitvec.h"
#include "catalog/index.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h"
#include "fmgr.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/sampling.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#include "vector.h"
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -63,13 +49,15 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/*
* Check with KMEANS_NORM_PROC that the value can be normalized since
* spherical distance function expects unit vectors
* Normalize with KMEANS_NORM_PROC since spherical distance function
* expects unit vectors
*/
if (buildstate->kmeansnormprocinfo != NULL)
{
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
return;
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
}
if (samples->length < targsamples)
@@ -80,7 +68,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
else
{
if (buildstate->rowstoskip < 0)
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, buildstate->samplerows, targsamples);
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
if (buildstate->rowstoskip <= 0)
{
@@ -96,9 +84,6 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
buildstate->rowstoskip -= 1;
}
/* Increment after reservoir_get_next_S */
buildstate->samplerows += 1;
}
/*
@@ -135,7 +120,6 @@ SampleRows(IvfflatBuildState * buildstate)
int targsamples = buildstate->samples->maxlen;
BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
buildstate->samplerows = 0;
buildstate->rowstoskip = -1;
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
@@ -148,28 +132,13 @@ SampleRows(IvfflatBuildState * buildstate)
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
}
/* Normalize if needed */
if (buildstate->kmeansnormprocinfo != NULL)
{
VectorArray samples = buildstate->samples;
for (int i = 0; i < samples->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(samples, i));
Datum normValue = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
VectorArraySet(samples, i, DatumGetPointer(normValue));
pfree(DatumGetPointer(normValue));
}
}
}
/*
* Add tuple to sort
*/
static void
AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
{
double distance;
double minDistance = DBL_MAX;
@@ -246,7 +215,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Add tuple to sort */
AddTupleToSort(tid, values, buildstate);
AddTupleToSort(index, tid, values, buildstate);
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
@@ -392,9 +361,6 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(buildstate->sortdesc);
#endif
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
@@ -456,7 +422,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
if (buildstate->heap != NULL)
{
IvfflatBench("sample rows", SampleRows(buildstate));
SampleRows(buildstate);
if (buildstate->samples->length < buildstate->lists)
{
@@ -504,8 +470,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum)
* Create list pages
*/
static void
CreateListPages(Relation index, VectorArray centers, int lists,
ForkNumber forkNum, ListInfo * *listInfo)
CreateListPages(Relation index, VectorArray centers, int dimensions,
int lists, ForkNumber forkNum, ListInfo * *listInfo)
{
Buffer buf;
Page page;
@@ -671,11 +637,7 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate);
buildstate.sortstate = ivfspool->sortstate;
scan = table_beginscan_parallel(ivfspool->heap,
ParallelTableScanFromIvfflatShared(ivfshared)
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
ParallelTableScanFromIvfflatShared(ivfshared));
reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo,
true, progress, BuildCallback,
(void *) &buildstate, scan);
@@ -1042,7 +1004,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
/* Create pages */
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo);
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum);
/* Write WAL for initialization fork since GenericXLog functions do not */
@@ -1061,10 +1023,6 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
IndexBuildResult *result;
IvfflatBuildState buildstate;
#ifdef IVFFLAT_BENCH
SeedRandom(42);
#endif
BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));

View File

@@ -3,19 +3,14 @@
#include <float.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/pg_list.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -97,7 +92,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Relation index;
/* Never use index without order */
if (path->indexorderbys == NIL)
if (path->indexorderbys == NULL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
@@ -184,64 +179,6 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
Datum
ivfflathandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 5,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = ivfflatbuild,
.ambuildempty = ivfflatbuildempty,
.aminsert = ivfflatinsert,
.aminsertcleanup = NULL,
.ambulkdelete = ivfflatbulkdelete,
.amvacuumcleanup = ivfflatvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = ivfflatcostestimate,
.amgettreeheight = NULL,
.amoptions = ivfflatoptions,
.amproperty = NULL,
.ambuildphasename = ivfflatbuildphasename,
.amvalidate = ivfflatvalidate,
.amadjustmembers = NULL,
.ambeginscan = ivfflatbeginscan,
.amrescan = ivfflatrescan,
.amgettuple = ivfflatgettuple,
.amgetbitmap = NULL,
.amendscan = ivfflatendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
@@ -249,11 +186,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -286,9 +218,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
amroutine->amcostestimate = ivfflatcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = ivfflatoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = ivfflatbuildphasename;
@@ -309,11 +238,5 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -9,15 +9,10 @@
#include "lib/pairingheap.h"
#include "nodes/execnodes.h"
#include "port.h" /* for random() */
#include "storage/condition_variable.h"
#include "utils/sampling.h"
#include "utils/tuplesort.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
@@ -26,10 +21,6 @@
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define IVFFLAT_MAX_DIM 2000
/* Support functions */
@@ -82,11 +73,9 @@ typedef Pointer Item;
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random()
#define SeedRandom(seed) srandom(seed)
#endif
/* Variables */
@@ -214,8 +203,7 @@ typedef struct IvfflatBuildState
/* Sampling */
BlockSamplerData bs;
ReservoirStateData rstate;
double samplerows;
double rowstoskip;
int rowstoskip;
/* Sorting */
Tuplesortstate *sortstate;

View File

@@ -2,16 +2,11 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
/*
* Find the list that minimizes the distance function
@@ -70,7 +65,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
* Insert a tuple into the index
*/
static void
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
{
const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
IndexTuple itup;
@@ -209,7 +204,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
oldCtx = MemoryContextSwitchTo(insertCtx);
/* Insert tuple */
InsertTuple(index, values, isnull, heap_tid);
InsertTuple(index, values, isnull, heap_tid, heap);
/* Delete memory context */
MemoryContextSwitchTo(oldCtx);

View File

@@ -1,19 +1,17 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "access/genam.h"
#include "fmgr.h"
#include "bitvec.h"
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#include "vector.h"
/*
* Initialize with kmeans++

View File

@@ -2,26 +2,15 @@
#include <float.h>
#include "access/genam.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/tupdesc.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "fmgr.h"
#include "lib/pairingheap.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
@@ -298,9 +287,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
so->tupdesc = CreateTemplateTupleDesc(2);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(so->tupdesc);
#endif
/* Prep sort */
so->sortstate = InitScanSortState(so->tupdesc);
@@ -369,10 +355,6 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)

View File

@@ -1,19 +1,13 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "bitvec.h"
#include "catalog/pg_type.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
#include "utils/relcache.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate a vector array
@@ -265,8 +259,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
for (int i = 0; i < dimensions; i++)
vec->x[i] = x[i];
for (int k = 0; k < dimensions; k++)
vec->x[k] = x[k];
}
static void
@@ -277,8 +271,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
for (int i = 0; i < dimensions; i++)
vec->x[i] = Float4ToHalfUnchecked(x[i]);
for (int k = 0; k < dimensions; k++)
vec->x[k] = Float4ToHalfUnchecked(x[k]);
}
static void
@@ -290,33 +284,29 @@ BitUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
for (uint32 i = 0; i < VARBITBYTES(vec); i++)
nx[i] = 0;
for (uint32 k = 0; k < VARBITBYTES(vec); k++)
nx[k] = 0;
for (int i = 0; i < dimensions; i++)
nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8));
for (int k = 0; k < dimensions; k++)
nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
}
static void
VectorSumCenter(Pointer v, float *x)
{
Vector *vec = (Vector *) v;
int dim = vec->dim;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
x[i] += vec->x[i];
for (int k = 0; k < vec->dim; k++)
x[k] += vec->x[k];
}
static void
HalfvecSumCenter(Pointer v, float *x)
{
HalfVector *vec = (HalfVector *) v;
int dim = vec->dim;
/* Auto-vectorized on aarch64 */
for (int i = 0; i < dim; i++)
x[i] += HalfToFloat4(vec->x[i]);
for (int k = 0; k < vec->dim; k++)
x[k] += HalfToFloat4(vec->x[k]);
}
static void
@@ -324,8 +314,8 @@ BitSumCenter(Pointer v, float *x)
{
VarBit *vec = (VarBit *) v;
for (int i = 0; i < VARBITLEN(vec); i++)
x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01);
for (int k = 0; k < VARBITLEN(vec); k++)
x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
}
/*

View File

@@ -1,16 +1,9 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "commands/vacuum.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/*
* Bulk delete tuples from the index

View File

@@ -5,27 +5,18 @@
#include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "common/string.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "lib/stringinfo.h"
#include "libpq/pqformat.h"
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
typedef struct SparseInputElement
{
int32 index;
@@ -159,9 +150,9 @@ InitSparseVector(int dim, int nnz)
return result;
}
#if PG_VERSION_NUM >= 170000
#define sparsevec_isspace(ch) scanner_isspace(ch)
#else
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
sparsevec_isspace(char ch)
{
@@ -174,7 +165,6 @@ sparsevec_isspace(char ch)
return true;
return false;
}
#endif
/*
* Compare indices

View File

@@ -16,20 +16,16 @@
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/varbit.h"
#include "utils/numeric.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -39,11 +35,7 @@
#define VECTOR_TARGET_CLONES
#endif
#if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.2");
#else
PG_MODULE_MAGIC;
#endif
/*
* Initialize index options and variables
@@ -133,9 +125,9 @@ InitVector(int dim)
return result;
}
#if PG_VERSION_NUM >= 170000
#define vector_isspace(ch) scanner_isspace(ch)
#else
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
vector_isspace(char ch)
{
@@ -148,7 +140,6 @@ vector_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -929,13 +920,11 @@ vector_concat(PG_FUNCTION_ARGS)
CheckDim(dim);
result = InitVector(dim);
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++)
for (int i = 0; i < a->dim; i++)
result->x[i] = a->x[i];
/* Auto-vectorized */
for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++)
result->x[i + start] = b->x[i];
for (int i = 0; i < b->dim; i++)
result->x[i + a->dim] = b->x[i];
PG_RETURN_POINTER(result);
}
@@ -951,21 +940,8 @@ binary_quantize(PG_FUNCTION_ARGS)
float *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
/* Auto-vectorized */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (ax[i + j] > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
for (int i = 0; i < a->dim; i++)
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);

View File

@@ -540,12 +540,6 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec)
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
subvector
-----------

View File

@@ -123,12 +123,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[0,0,0]
(3 rows)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET hnsw.iterative_scan;
RESET hnsw.ef_search;
DROP TABLE t;

View File

@@ -110,15 +110,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[1,1,1]
(2 rows)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes;
DROP TABLE t;

View File

@@ -576,12 +576,6 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
subvector
-----------

View File

@@ -121,7 +121,6 @@ SELECT l2_normalize('[65504]'::halfvec);
SELECT binary_quantize('[1,0,-1]'::halfvec);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);

View File

@@ -70,9 +70,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SET hnsw.iterative_scan = relaxed_order;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
RESET hnsw.iterative_scan;
RESET hnsw.ef_search;
DROP TABLE t;

View File

@@ -59,9 +59,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SET ivfflat.max_probes = 2;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes;
DROP TABLE t;

View File

@@ -128,7 +128,6 @@ SELECT l2_normalize('[3e38]'::vector);
SELECT binary_quantize('[1,0,-1]'::vector);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);

View File

@@ -16,19 +16,29 @@ $node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
# Check each index type
my @operators = ("<->", "<#>", "<=>");
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
# Test 100% recall
for (1 .. 20)
for my $i (0 .. $#operators)
{
my $id = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test 100% recall
for (1 .. 20)
{
my $id = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
}
done_testing();

View File

@@ -1,29 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 1000) i;"
);
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '3073kB';
ALTER TABLE tst SET (parallel_workers = 1);
CREATE INDEX ON tst USING hnsw (v vector_l2_ops);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem after 0 tuples/);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.2'
default_version = '0.8.0'
module_pathname = '$libdir/vector'
relocatable = true