Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
ec68b75892 Added bulk hashing [skip ci] 2024-10-29 11:41:57 -07:00
27 changed files with 187 additions and 279 deletions

View File

@@ -1,15 +1,8 @@
/.git/ /.git/
/dist/ /dist/
/log/
/results/ /results/
/tmp_check/ /tmp_check/
/sql/vector--?.?.?.sql /sql/vector--?.?.?.sql
regression.* regression.*
*.o *.o
*.so *.so
*.bc
*.dll
*.dylib
*.obj
*.lib
*.exp

View File

@@ -13,13 +13,13 @@ jobs:
- postgres: 17 - postgres: 17
os: ubuntu-24.04 os: ubuntu-24.04
- postgres: 16 - postgres: 16
os: ubuntu-24.04-arm os: ubuntu-22.04
- postgres: 15 - postgres: 15
os: ubuntu-22.04 os: ubuntu-22.04
- postgres: 14 - postgres: 14
os: ubuntu-22.04-arm os: ubuntu-20.04
- postgres: 13 - postgres: 13
os: ubuntu-22.04 os: ubuntu-20.04
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
@@ -28,7 +28,7 @@ jobs:
dev-files: true dev-files: true
- run: make - run: make
env: env:
PG_CFLAGS: ${{ matrix.postgres == 18 && '-Wno-missing-field-initializers' || '' }} -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: | - run: |
export PG_CONFIG=`which pg_config` export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install sudo --preserve-env=PG_CONFIG make install
@@ -46,8 +46,8 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- postgres: 17 - postgres: 16
os: macos-15 os: macos-14
- postgres: 14 - postgres: 14
os: macos-13 os: macos-13
steps: steps:
@@ -57,7 +57,7 @@ jobs:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
- run: make - run: make
env: env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install - run: make install
- run: make installcheck - run: make installcheck
- if: ${{ failure() }} - if: ${{ failure() }}
@@ -70,13 +70,12 @@ jobs:
tar xf $TAG.tar.gz tar xf $TAG.tar.gz
mv postgres-$TAG postgres mv postgres-$TAG postgres
env: env:
TAG: ${{ matrix.postgres == 17 && 'REL_17_2' || 'REL_14_15' }} TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl" - run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env: env:
PERL5LIB: /Users/runner/perl5/lib/perl5 PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make - run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
env: env:
LLVM_VERSION: ${{ matrix.os == 'macos-15' && 18 || 15 }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows: windows:
runs-on: windows-latest runs-on: windows-latest
@@ -126,7 +125,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1 - uses: ankane/setup-postgres-valgrind@v1
with: with:
postgres-version: 17 postgres-version: 16
check-ub: yes check-ub: yes
- run: make OPTFLAGS="" - run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install - run: sudo --preserve-env=PG_CONFIG make install

View File

@@ -1,8 +1,4 @@
## 0.8.1 (unreleased) ## 0.8.0 (unreleased)
- Improved performance of HNSW index scans for Postgres 17
## 0.8.0 (2024-10-30)
- Added support for iterative index scans - Added support for iterative index scans
- Added casts for arrays to `sparsevec` - Added casts for arrays to `sparsevec`

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.0", "version": "0.7.4",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -12,7 +12,7 @@
"prereqs": { "prereqs": {
"runtime": { "runtime": {
"requires": { "requires": {
"PostgreSQL": "13.0.0" "PostgreSQL": "12.0.0"
} }
} }
}, },
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.8.0", "version": "0.7.4",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.0 EXTVERSION = 0.7.4
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql) DATA = $(wildcard sql/*--*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.0 EXTVERSION = 0.7.4
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

151
README.md
View File

@@ -17,11 +17,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
### Linux and Mac ### Linux and Mac
Compile and install the extension (supports Postgres 13+) Compile and install the extension (supports Postgres 12+)
```sh ```sh
cd /tmp cd /tmp
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -33,17 +33,27 @@ You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#p
### Windows ### Windows
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build: Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
```cmd ```cmd
set "PGROOT=C:\Program Files\PostgreSQL\17" call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
```
Note: The exact path will vary depending on your Visual Studio version and edition
Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP% cd %TEMP%
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
nmake /F Makefile.win nmake /F Makefile.win
nmake /F Makefile.win install nmake /F Makefile.win install
``` ```
Note: Postgres 17 is not supported yet due to an upstream issue
See the [installation notes](#installation-notes---windows) if you run into issues See the [installation notes](#installation-notes---windows) if you run into issues
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge). You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -74,7 +84,7 @@ Get the nearest neighbors by L2 distance
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
``` ```
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`) Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
@@ -138,9 +148,9 @@ Supported distance functions are:
- `<->` - L2 distance - `<->` - L2 distance
- `<#>` - (negative) inner product - `<#>` - (negative) inner product
- `<=>` - cosine distance - `<=>` - cosine distance
- `<+>` - L1 distance - `<+>` - L1 distance (added in 0.7.0)
- `<~>` - Hamming distance (binary vectors) - `<~>` - Hamming distance (binary vectors, added in 0.7.0)
- `<%>` - Jaccard distance (binary vectors) - `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
Get the nearest neighbors to a row Get the nearest neighbors to a row
@@ -227,19 +237,19 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops); CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
``` ```
L1 distance L1 distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops); CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
``` ```
Hamming distance Hamming distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops); CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
``` ```
Jaccard distance Jaccard distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops); CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
@@ -248,9 +258,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
Supported types are: Supported types are:
- `vector` - up to 2,000 dimensions - `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions - `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions - `bit` - up to 64,000 dimensions (added in 0.7.0)
- `sparsevec` - up to 1,000 non-zero elements - `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0)
### Index Options ### Index Options
@@ -304,19 +314,17 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t
Like other index types, its faster to create an index after loading your initial data Like other index types, its faster to create an index after loading your initial data
You can also speed up index creation by increasing the number of parallel workers (2 by default) Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql ```sql
SET max_parallel_maintenance_workers = 7; -- plus leader SET max_parallel_maintenance_workers = 7; -- plus leader
``` ```
For a large number of workers, you may need to increase `max_parallel_workers` (8 by default) For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
### Indexing Progress ### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
```sql ```sql
SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index; SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -359,7 +367,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
``` ```
Hamming distance Hamming distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
@@ -368,8 +376,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
Supported types are: Supported types are:
- `vector` - up to 2,000 dimensions - `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions - `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions - `bit` - up to 64,000 dimensions (added in 0.7.0)
### Query Options ### Query Options
@@ -402,7 +410,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
### Indexing Progress ### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
```sql ```sql
SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index; SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -469,6 +477,8 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
## Iterative Index Scans ## Iterative Index Scans
*Unreleased*
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
Iterative scans can use strict or relaxed ordering. Iterative scans can use strict or relaxed ordering.
@@ -492,11 +502,9 @@ With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.o
```sql ```sql
WITH relaxed_results AS MATERIALIZED ( WITH relaxed_results AS MATERIALIZED (
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5 SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
) SELECT * FROM relaxed_results ORDER BY distance + 0; ) SELECT * FROM relaxed_results ORDER BY distance;
``` ```
Note: `+ 0` is needed for Postgres 17+
For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor) For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
```sql ```sql
@@ -541,6 +549,8 @@ Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used
## Half-Precision Vectors ## Half-Precision Vectors
*Added in 0.7.0*
Use the `halfvec` type to store half-precision vectors Use the `halfvec` type to store half-precision vectors
```sql ```sql
@@ -549,6 +559,8 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
## Half-Precision Indexing ## Half-Precision Indexing
*Added in 0.7.0*
Index vectors at half precision for smaller indexes Index vectors at half precision for smaller indexes
```sql ```sql
@@ -570,16 +582,24 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111'); INSERT INTO items (embedding) VALUES ('000'), ('111');
``` ```
Get the nearest neighbors by Hamming distance Get the nearest neighbors by Hamming distance (added in 0.7.0)
```sql ```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5; SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
``` ```
Or (before 0.7.0)
```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
```
Also supports Jaccard distance (`<%>`) Also supports Jaccard distance (`<%>`)
## Binary Quantization ## Binary Quantization
*Added in 0.7.0*
Use expression indexing for binary quantization Use expression indexing for binary quantization
```sql ```sql
@@ -602,6 +622,8 @@ SELECT * FROM (
## Sparse Vectors ## Sparse Vectors
*Added in 0.7.0*
Use the `sparsevec` type to store sparse vectors Use the `sparsevec` type to store sparse vectors
```sql ```sql
@@ -635,6 +657,8 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
## Indexing Subvectors ## Indexing Subvectors
*Added in 0.7.0*
Use expression indexing to index subvectors Use expression indexing to index subvectors
```sql ```sql
@@ -748,6 +772,8 @@ SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_t
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20; FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
``` ```
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
Monitor recall by comparing results from approximate search with exact search. Monitor recall by comparing results from approximate search with exact search.
```sql ```sql
@@ -775,17 +801,13 @@ C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp) C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet) C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal) Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart) Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir) Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
Erlang | [pgvector-erlang](https://github.com/pgvector/pgvector-erlang)
Fortran | [pgvector-fortran](https://github.com/pgvector/pgvector-fortran)
Gleam | [pgvector-gleam](https://github.com/pgvector/pgvector-gleam)
Go | [pgvector-go](https://github.com/pgvector/pgvector-go) Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell) Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java) Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node) JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl) Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp) Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua) Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim) Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
@@ -794,7 +816,6 @@ Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php) PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python) Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r) R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor) Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust) Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift) Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
@@ -816,7 +837,7 @@ You can use [half-precision indexing](#half-precision-indexing) to index up to 4
#### Can I store vectors with different dimensions in the same column? #### Can I store vectors with different dimensions in the same column?
You can use `vector` as the type (instead of `vector(n)`). You can use `vector` as the type (instead of `vector(3)`).
```sql ```sql
CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id)); CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
@@ -916,7 +937,7 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
#### Why are there less results for a query after adding an HNSW index? #### Why are there less results for a query after adding an HNSW index?
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`), which is 40 by default. There may be even less results due to dead tuples or filtering conditions in the query. Enabling [iterative index scans](#iterative-index-scans) can help address this. Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance). Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -928,7 +949,7 @@ The index was likely created with too little data for the number of lists. Drop
DROP INDEX index_name; DROP INDEX index_name;
``` ```
Results can also be limited by the number of probes (`ivfflat.probes`). Enabling [iterative index scans](#iterative-index-scans) can address this. Results can also be limited by the number of probes (`ivfflat.probes`).
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance). Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -1095,13 +1116,7 @@ Note: Replace `17` with your Postgres server version
### Missing SDK ### Missing SDK
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists. If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
```sh
pg_config --cppflags
```
Reinstall Postgres to fix this.
### Portability ### Portability
@@ -1119,14 +1134,6 @@ make OPTFLAGS=""
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct. If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
### Mismatched Architecture
If compilation fails with `error C2196: case value '4' already used`, make sure youre using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
### Missing Symbol
If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+.
### Permissions ### Permissions
If installation fails with `Access is denied`, re-run the installation instructions as an administrator. If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
@@ -1146,17 +1153,11 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually: You can also build the image manually:
```sh ```sh
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector . docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
``` ```
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
```
### Homebrew ### Homebrew
With Homebrew Postgres, you can use: With Homebrew Postgres, you can use:
@@ -1202,7 +1203,7 @@ Note: Replace `17` with your Postgres server version
Install the FreeBSD package with: Install the FreeBSD package with:
```sh ```sh
pkg install postgresql16-pgvector pkg install postgresql15-pgvector
``` ```
or the port with: or the port with:
@@ -1244,6 +1245,36 @@ You can check the version in the current database with:
SELECT extversion FROM pg_extension WHERE extname = 'vector'; SELECT extversion FROM pg_extension WHERE extname = 'vector';
``` ```
## Upgrade Notes
### 0.6.0
#### Postgres 12
If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
```sql
ALTER TYPE vector SET (STORAGE = external);
```
Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
#### Docker
The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
```sh
docker pull pgvector/pgvector:pg16
# or
docker pull pgvector/pgvector:0.6.0-pg16
```
Also, if youve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
```
## Thanks ## Thanks
Thanks to: Thanks to:

View File

@@ -259,11 +259,6 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0; amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false; amroutine->amcanorder = false;
amroutine->amcanorderbyop = true; amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */ amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false; amroutine->amcanunique = false;
amroutine->amcanmulticol = false; amroutine->amcanmulticol = false;
@@ -296,9 +291,6 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = hnswvacuumcleanup; amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL; amroutine->amcanreturn = NULL;
amroutine->amcostestimate = hnswcostestimate; amroutine->amcostestimate = hnswcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = hnswoptions; amroutine->amoptions = hnswoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */ amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = hnswbuildphasename; amroutine->ambuildphasename = hnswbuildphasename;
@@ -319,10 +311,5 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL; amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL; amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine); PG_RETURN_POINTER(amroutine);
} }

View File

@@ -126,7 +126,7 @@ typedef struct HnswNeighborArray HnswNeighborArray;
#define HnswPtrDeclare(type, relptrtype, ptrtype) \ #define HnswPtrDeclare(type, relptrtype, ptrtype) \
relptr_declare(type, relptrtype); \ relptr_declare(type, relptrtype); \
typedef union { type *ptr; relptrtype relptr; } ptrtype typedef union { type *ptr; relptrtype relptr; } ptrtype;
/* Pointers that can be absolute or relative */ /* Pointers that can be absolute or relative */
/* Use char for DatumPtr so works with Pointer */ /* Use char for DatumPtr so works with Pointer */
@@ -362,13 +362,6 @@ typedef union
ItemPointerData indextid; ItemPointerData indextid;
} HnswUnvisited; } HnswUnvisited;
typedef struct HnswReadStreamData
{
HnswUnvisited *unvisited;
int unvisitedLength;
int visited;
} HnswReadStreamData;
typedef struct HnswScanOpaqueData typedef struct HnswScanOpaqueData
{ {
const HnswTypeInfo *typeInfo; const HnswTypeInfo *typeInfo;
@@ -424,13 +417,13 @@ bool HnswCheckNorm(HnswSupport * support, Datum value);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum); Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page); void HnswInitPage(Buffer buf, Page page);
void HnswInit(void); void HnswInit(void);
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples, bool maintenance); List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
HnswElement HnswGetEntryPoint(Relation index); HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint); void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
void *HnswAlloc(HnswAllocator * allocator, Size size); void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc); HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno); HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing, bool maintenance); void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec); HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building); void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m); void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);

View File

@@ -457,7 +457,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
} }
/* Find neighbors for element */ /* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false, true); HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
/* Update graph in memory */ /* Update graph in memory */
UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate); UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate);

View File

@@ -725,7 +725,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
} }
/* Find neighbors for element */ /* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false, building); HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
/* Update graph on disk */ /* Update graph on disk */
UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building); UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building);

View File

@@ -37,11 +37,11 @@ GetScanItems(IndexScanDesc scan, Datum value)
for (int lc = entryPoint->level; lc >= 1; lc--) for (int lc = entryPoint->level; lc >= 1; lc--)
{ {
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL, false); w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL);
ep = w; ep = w;
} }
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples, false); return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples);
} }
/* /*
@@ -72,7 +72,7 @@ ResumeScanItems(IndexScanDesc scan)
ep = lappend(ep, sc); ep = lappend(ep, sc);
} }
return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples, false); return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
} }
/* /*

View File

@@ -15,10 +15,6 @@
#include "utils/memdebug.h" #include "utils/memdebug.h"
#include "utils/rel.h" #include "utils/rel.h"
#if PG_VERSION_NUM >= 170000
#include "storage/read_stream.h"
#endif
#if PG_VERSION_NUM < 170000 #if PG_VERSION_NUM < 170000
static inline uint64 static inline uint64
murmurhash64(uint64 data) murmurhash64(uint64 data)
@@ -529,12 +525,14 @@ HnswGetDistance(Datum a, Datum b, HnswSupport * support)
* Load an element and optionally get its distance from q * Load an element and optionally get its distance from q
*/ */
static void static void
HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance, HnswElement * element) HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance, HnswElement * element)
{ {
Buffer buf;
Page page; Page page;
HnswElementTuple etup; HnswElementTuple etup;
/* Read vector */ /* Read vector */
buf = ReadBuffer(index, blkno);
LockBuffer(buf, BUFFER_LOCK_SHARE); LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf); page = BufferGetPage(buf);
@@ -555,7 +553,7 @@ HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery
if (distance == NULL || maxDistance == NULL || *distance < *maxDistance) if (distance == NULL || maxDistance == NULL || *distance < *maxDistance)
{ {
if (*element == NULL) if (*element == NULL)
*element = HnswInitElementFromBlock(BufferGetBlockNumber(buf), offno); *element = HnswInitElementFromBlock(blkno, offno);
HnswLoadElementFromTuple(*element, etup, true, loadVec); HnswLoadElementFromTuple(*element, etup, true, loadVec);
} }
@@ -569,9 +567,7 @@ HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery
void void
HnswLoadElement(HnswElement element, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance) HnswLoadElement(HnswElement element, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance)
{ {
Buffer buf = ReadBuffer(index, element->blkno); HnswLoadElementImpl(element->blkno, element->offno, distance, q, index, support, loadVec, maxDistance, &element);
HnswLoadElementImpl(buf, element->offno, distance, q, index, support, loadVec, maxDistance, &element);
} }
/* /*
@@ -726,6 +722,8 @@ CountElement(HnswElement skipElement, HnswElement e)
static void static void
HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize) HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize)
{ {
uint32 hashes[HNSW_MAX_M * 2];
/* Get the neighborhood at layer lc */ /* Get the neighborhood at layer lc */
HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc); HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc);
@@ -736,16 +734,35 @@ HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unv
*unvisitedLength = 0; *unvisitedLength = 0;
for (int i = 0; i < localNeighborhood->length; i++)
hashes[i] = HnswPtrAccess(base, localNeighborhood->items[i].element)->hash;
if (base != NULL)
{
for (int i = 0; i < localNeighborhood->length; i++) for (int i = 0; i < localNeighborhood->length; i++)
{ {
HnswCandidate *hc = &localNeighborhood->items[i]; HnswCandidate *hc = &localNeighborhood->items[i];
bool found; bool found;
AddToVisited(base, v, hc->element, true, &found); offsethash_insert_hash(v->offsets, HnswPtrOffset(hc->element), hashes[i], &found);
if (!found) if (!found)
unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element); unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
} }
}
else
{
for (int i = 0; i < localNeighborhood->length; i++)
{
HnswCandidate *hc = &localNeighborhood->items[i];
bool found;
pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), hashes[i], &found);
if (!found)
unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
}
}
} }
/* /*
@@ -811,31 +828,11 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
} }
} }
#if PG_VERSION_NUM >= 170000
/*
* Get next block number for read stream
*/
static BlockNumber
HnswReadStreamNextBlock(ReadStream *stream, void *callback_private_data, void *per_buffer_data)
{
HnswReadStreamData *streamData = callback_private_data;
OffsetNumber *offno = per_buffer_data;
HnswUnvisited *uv;
if (streamData->visited == streamData->unvisitedLength)
return InvalidBlockNumber;
uv = &streamData->unvisited[streamData->visited++];
*offno = ItemPointerGetOffsetNumber(&uv->indextid);
return ItemPointerGetBlockNumber(&uv->indextid);
}
#endif
/* /*
* Algorithm 2 from paper * Algorithm 2 from paper
*/ */
List * List *
HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples, bool maintenance) HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples)
{ {
List *w = NIL; List *w = NIL;
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL); pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
@@ -850,18 +847,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
int unvisitedLength; int unvisitedLength;
bool inMemory = index == NULL; bool inMemory = index == NULL;
#if PG_VERSION_NUM >= 170000
HnswReadStreamData streamData;
ReadStream *stream = NULL;
if (!inMemory)
{
int flags = maintenance ? READ_STREAM_MAINTENANCE : READ_STREAM_DEFAULT;
stream = read_stream_begin_relation(flags, NULL, index, MAIN_FORKNUM, HnswReadStreamNextBlock, &streamData, sizeof(OffsetNumber));
}
#endif
if (v == NULL) if (v == NULL)
{ {
v = &vh; v = &vh;
@@ -924,18 +909,8 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
if (inMemory) if (inMemory)
HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize); HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize);
else else
{
HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc); HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
#if PG_VERSION_NUM >= 170000
read_stream_reset(stream);
streamData.unvisited = unvisited;
streamData.unvisitedLength = unvisitedLength;
streamData.visited = 0;
#endif
}
/* OK to count elements instead of tuples */ /* OK to count elements instead of tuples */
if (tuples != NULL) if (tuples != NULL)
(*tuples) += unvisitedLength; (*tuples) += unvisitedLength;
@@ -956,24 +931,13 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
} }
else else
{ {
Buffer buf;
OffsetNumber offno;
#if PG_VERSION_NUM >= 170000
void *offnoPtr;
buf = read_stream_next_buffer(stream, &offnoPtr);
offno = *((OffsetNumber *) offnoPtr);
#else
ItemPointer indextid = &unvisited[i].indextid; ItemPointer indextid = &unvisited[i].indextid;
BlockNumber blkno = ItemPointerGetBlockNumber(indextid);
buf = ReadBuffer(index, ItemPointerGetBlockNumber(indextid)); OffsetNumber offno = ItemPointerGetOffsetNumber(indextid);
offno = ItemPointerGetOffsetNumber(indextid);
#endif
/* Avoid any allocations if not adding */ /* Avoid any allocations if not adding */
eElement = NULL; eElement = NULL;
HnswLoadElementImpl(buf, offno, &eDistance, q, index, support, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement); HnswLoadElementImpl(blkno, offno, &eDistance, q, index, support, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
if (eElement == NULL) if (eElement == NULL)
continue; continue;
@@ -1029,11 +993,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
w = lappend(w, sc); w = lappend(w, sc);
} }
#if PG_VERSION_NUM >= 170000
if (!inMemory)
read_stream_end(stream);
#endif
return w; return w;
} }
@@ -1329,7 +1288,7 @@ PrecomputeHash(char *base, HnswElement element)
* Algorithm 1 from paper * Algorithm 1 from paper
*/ */
void void
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing, bool maintenance) HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing)
{ {
List *ep; List *ep;
List *w; List *w;
@@ -1356,7 +1315,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
/* 1st phase: greedy search to insert level */ /* 1st phase: greedy search to insert level */
for (int lc = entryLevel; lc >= level + 1; lc--) for (int lc = entryLevel; lc >= level + 1; lc--)
{ {
w = HnswSearchLayer(base, &q, ep, 1, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL, maintenance); w = HnswSearchLayer(base, &q, ep, 1, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL);
ep = w; ep = w;
} }
@@ -1375,7 +1334,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
List *lw = NIL; List *lw = NIL;
ListCell *lc2; ListCell *lc2;
w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL, maintenance); w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL);
/* Convert search candidates to candidates */ /* Convert search candidates to candidates */
foreach(lc2, w) foreach(lc2, w)
@@ -1455,7 +1414,7 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support); FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
Datum Datum
@@ -1468,7 +1427,7 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support); FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
Datum Datum
@@ -1481,4 +1440,4 @@ hnsw_sparsevec_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };

View File

@@ -9,10 +9,6 @@
#include "storage/lmgr.h" #include "storage/lmgr.h"
#include "utils/memutils.h" #include "utils/memutils.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/* /*
* Check if deleted list contains an index TID * Check if deleted list contains an index TID
*/ */
@@ -208,7 +204,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
element->heaptidsLength = 0; element->heaptidsLength = 0;
/* Find neighbors for element, skipping itself */ /* Find neighbors for element, skipping itself */
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true, true); HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true);
/* Zero memory for each element */ /* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE); MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);

View File

@@ -360,7 +360,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->sortdesc = CreateTemplateTupleDesc(3); buildstate->sortdesc = CreateTemplateTupleDesc(3);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", buildstate->tupdesc->attrs[0].atttypid, -1, 0);
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual); buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
@@ -1023,10 +1023,6 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
IndexBuildResult *result; IndexBuildResult *result;
IvfflatBuildState buildstate; IvfflatBuildState buildstate;
#ifdef IVFFLAT_BENCH
SeedRandom(42);
#endif
BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM); BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult)); result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));

View File

@@ -186,11 +186,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0; amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false; amroutine->amcanorder = false;
amroutine->amcanorderbyop = true; amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */ amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false; amroutine->amcanunique = false;
amroutine->amcanmulticol = false; amroutine->amcanmulticol = false;
@@ -223,9 +218,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = ivfflatvacuumcleanup; amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */ amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
amroutine->amcostestimate = ivfflatcostestimate; amroutine->amcostestimate = ivfflatcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = ivfflatoptions; amroutine->amoptions = ivfflatoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */ amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = ivfflatbuildphasename; amroutine->ambuildphasename = ivfflatbuildphasename;
@@ -246,10 +238,5 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL; amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL; amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine); PG_RETURN_POINTER(amroutine);
} }

View File

@@ -73,11 +73,9 @@
#if PG_VERSION_NUM >= 150000 #if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state) #define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state) #define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
#else #else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE) #define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random() #define RandomInt() random()
#define SeedRandom(seed) srandom(seed)
#endif #endif
/* Variables */ /* Variables */

View File

@@ -259,8 +259,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VECTOR_SIZE(dimensions)); SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions; vec->dim = dimensions;
for (int i = 0; i < dimensions; i++) for (int k = 0; k < dimensions; k++)
vec->x[i] = x[i]; vec->x[k] = x[k];
} }
static void static void
@@ -271,8 +271,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions)); SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions; vec->dim = dimensions;
for (int i = 0; i < dimensions; i++) for (int k = 0; k < dimensions; k++)
vec->x[i] = Float4ToHalfUnchecked(x[i]); vec->x[k] = Float4ToHalfUnchecked(x[k]);
} }
static void static void
@@ -284,33 +284,29 @@ BitUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions)); SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions; VARBITLEN(vec) = dimensions;
for (uint32 i = 0; i < VARBITBYTES(vec); i++) for (uint32 k = 0; k < VARBITBYTES(vec); k++)
nx[i] = 0; nx[k] = 0;
for (int i = 0; i < dimensions; i++) for (int k = 0; k < dimensions; k++)
nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8)); nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
} }
static void static void
VectorSumCenter(Pointer v, float *x) VectorSumCenter(Pointer v, float *x)
{ {
Vector *vec = (Vector *) v; Vector *vec = (Vector *) v;
int dim = vec->dim;
/* Auto-vectorized */ for (int k = 0; k < vec->dim; k++)
for (int i = 0; i < dim; i++) x[k] += vec->x[k];
x[i] += vec->x[i];
} }
static void static void
HalfvecSumCenter(Pointer v, float *x) HalfvecSumCenter(Pointer v, float *x)
{ {
HalfVector *vec = (HalfVector *) v; HalfVector *vec = (HalfVector *) v;
int dim = vec->dim;
/* Auto-vectorized on aarch64 */ for (int k = 0; k < vec->dim; k++)
for (int i = 0; i < dim; i++) x[k] += HalfToFloat4(vec->x[k]);
x[i] += HalfToFloat4(vec->x[i]);
} }
static void static void
@@ -318,8 +314,8 @@ BitSumCenter(Pointer v, float *x)
{ {
VarBit *vec = (VarBit *) v; VarBit *vec = (VarBit *) v;
for (int i = 0; i < VARBITLEN(vec); i++) for (int k = 0; k < VARBITLEN(vec); k++)
x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01); x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
} }
/* /*
@@ -359,7 +355,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support); FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
Datum Datum
@@ -374,4 +370,4 @@ ivfflat_bit_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };

View File

@@ -5,10 +5,6 @@
#include "ivfflat.h" #include "ivfflat.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/* /*
* Bulk delete tuples from the index * Bulk delete tuples from the index
*/ */

View File

@@ -4,7 +4,6 @@
#include <math.h> #include <math.h>
#include "catalog/pg_type.h" #include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "common/string.h" #include "common/string.h"
#include "fmgr.h" #include "fmgr.h"
#include "halfutils.h" #include "halfutils.h"
@@ -13,10 +12,17 @@
#include "sparsevec.h" #include "sparsevec.h"
#include "utils/array.h" #include "utils/array.h"
#include "utils/builtins.h" #include "utils/builtins.h"
#include "utils/float.h"
#include "utils/lsyscache.h" #include "utils/lsyscache.h"
#include "vector.h" #include "vector.h"
#if PG_VERSION_NUM >= 120000
#include "common/shortest_dec.h"
#include "utils/float.h"
#else
#include <float.h>
#include "utils/builtins.h"
#endif
typedef struct SparseInputElement typedef struct SparseInputElement
{ {
int32 index; int32 index;

View File

@@ -916,19 +916,15 @@ vector_concat(PG_FUNCTION_ARGS)
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
Vector *result; Vector *result;
int dim = a->dim + b->dim; int dim = a->dim + b->dim;
int dim_a = a->dim;
int dim_b = b->dim;
CheckDim(dim); CheckDim(dim);
result = InitVector(dim); result = InitVector(dim);
/* Auto-vectorized */ for (int i = 0; i < a->dim; i++)
for (int i = 0; i < dim_a; i++)
result->x[i] = a->x[i]; result->x[i] = a->x[i];
/* Auto-vectorized */ for (int i = 0; i < b->dim; i++)
for (int i = 0; i < dim_b; i++) result->x[i + a->dim] = b->x[i];
result->x[i + dim_a] = b->x[i];
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }

View File

@@ -123,12 +123,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[0,0,0] [0,0,0]
(3 rows) (3 rows)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET hnsw.iterative_scan; RESET hnsw.iterative_scan;
RESET hnsw.ef_search; RESET hnsw.ef_search;
DROP TABLE t; DROP TABLE t;

View File

@@ -110,15 +110,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[1,1,1] [1,1,1]
(2 rows) (2 rows)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET ivfflat.iterative_scan; RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes; RESET ivfflat.max_probes;
DROP TABLE t; DROP TABLE t;

View File

@@ -70,9 +70,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SET hnsw.iterative_scan = relaxed_order; SET hnsw.iterative_scan = relaxed_order;
SELECT * FROM t ORDER BY val <-> '[3,3,3]'; SELECT * FROM t ORDER BY val <-> '[3,3,3]';
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
RESET hnsw.iterative_scan; RESET hnsw.iterative_scan;
RESET hnsw.ef_search; RESET hnsw.ef_search;
DROP TABLE t; DROP TABLE t;

View File

@@ -59,9 +59,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SET ivfflat.max_probes = 2; SET ivfflat.max_probes = 2;
SELECT * FROM t ORDER BY val <-> '[3,3,3]'; SELECT * FROM t ORDER BY val <-> '[3,3,3]';
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
RESET ivfflat.iterative_scan; RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes; RESET ivfflat.max_probes;
DROP TABLE t; DROP TABLE t;

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods' comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.0' default_version = '0.7.4'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true