mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
1 Commits
hnsw-read-
...
guc-explai
| Author | SHA1 | Date | |
|---|---|---|---|
|
fc0d3e7fdb |
8
.dockerignore
Normal file
8
.dockerignore
Normal file
@@ -0,0 +1,8 @@
|
||||
/.git/
|
||||
/dist/
|
||||
/results/
|
||||
/tmp_check/
|
||||
/sql/vector--?.?.?.sql
|
||||
regression.*
|
||||
*.o
|
||||
*.so
|
||||
47
.github/workflows/build.yml
vendored
47
.github/workflows/build.yml
vendored
@@ -8,29 +8,27 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- postgres: 19
|
||||
os: ubuntu-24.04
|
||||
- postgres: 18
|
||||
os: ubuntu-24.04
|
||||
- postgres: 17
|
||||
os: ubuntu-24.04
|
||||
- postgres: 16
|
||||
os: ubuntu-24.04-arm
|
||||
os: ubuntu-22.04
|
||||
- postgres: 15
|
||||
os: ubuntu-22.04
|
||||
- postgres: 14
|
||||
os: ubuntu-22.04-arm
|
||||
os: ubuntu-20.04
|
||||
- postgres: 13
|
||||
os: ubuntu-22.04
|
||||
os: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres@v1
|
||||
with:
|
||||
postgres-version: ${{ matrix.postgres }}
|
||||
dev-files: true
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
- run: |
|
||||
export PG_CONFIG=`which pg_config`
|
||||
sudo --preserve-env=PG_CONFIG make install
|
||||
@@ -48,18 +46,18 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- postgres: 18
|
||||
os: macos-26
|
||||
- postgres: 16
|
||||
os: macos-14
|
||||
- postgres: 14
|
||||
os: macos-15-intel
|
||||
os: macos-13
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres@v1
|
||||
with:
|
||||
postgres-version: ${{ matrix.postgres }}
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
|
||||
- run: make install
|
||||
- run: make installcheck
|
||||
- if: ${{ failure() }}
|
||||
@@ -72,35 +70,26 @@ jobs:
|
||||
tar xf $TAG.tar.gz
|
||||
mv postgres-$TAG postgres
|
||||
env:
|
||||
TAG: ${{ matrix.postgres == 18 && 'REL_18_0' || 'REL_14_19' }}
|
||||
TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
|
||||
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
|
||||
env:
|
||||
PERL5LIB: /Users/runner/perl5/lib/perl5
|
||||
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make
|
||||
- run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
|
||||
env:
|
||||
LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }}
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING
|
||||
windows:
|
||||
runs-on: ${{ matrix.os }}
|
||||
runs-on: windows-latest
|
||||
if: ${{ !startsWith(github.ref_name, 'mac') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- postgres: 17
|
||||
os: windows-2025
|
||||
- postgres: 14
|
||||
os: windows-2022
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres@v1
|
||||
with:
|
||||
postgres-version: ${{ matrix.postgres }}
|
||||
postgres-version: 14
|
||||
- run: |
|
||||
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
|
||||
nmake /NOLOGO /F Makefile.win && ^
|
||||
nmake /NOLOGO /F Makefile.win install && ^
|
||||
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
|
||||
nmake /NOLOGO /F Makefile.win installcheck && ^
|
||||
nmake /NOLOGO /F Makefile.win clean && ^
|
||||
nmake /NOLOGO /F Makefile.win uninstall
|
||||
shell: cmd
|
||||
@@ -133,10 +122,10 @@ jobs:
|
||||
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres-valgrind@v1
|
||||
with:
|
||||
postgres-version: 18
|
||||
postgres-version: 16
|
||||
check-ub: yes
|
||||
- run: make OPTFLAGS=""
|
||||
- run: sudo --preserve-env=PG_CONFIG make install
|
||||
|
||||
16
CHANGELOG.md
16
CHANGELOG.md
@@ -1,20 +1,10 @@
|
||||
## 0.8.2 (unreleased)
|
||||
|
||||
- Improved `install` target on Windows
|
||||
- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
|
||||
|
||||
## 0.8.1 (2025-09-04)
|
||||
|
||||
- Added support for Postgres 18 rc1
|
||||
- Improved performance of `binary_quantize` function
|
||||
|
||||
## 0.8.0 (2024-10-30)
|
||||
## 0.8.0 (unreleased)
|
||||
|
||||
- Added support for iterative index scans
|
||||
- Added casts for arrays to `sparsevec`
|
||||
- Improved cost estimation for better index selection when filtering
|
||||
- Improved performance of HNSW index scans
|
||||
- Improved cost estimation
|
||||
- Improved performance of HNSW inserts and on-disk index builds
|
||||
- Reduced memory usage for HNSW index scans
|
||||
- Dropped support for Postgres 12
|
||||
|
||||
## 0.7.4 (2024-08-05)
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
ARG PG_MAJOR=17
|
||||
ARG DEBIAN_CODENAME=bookworm
|
||||
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
|
||||
FROM postgres:$PG_MAJOR
|
||||
ARG PG_MAJOR
|
||||
|
||||
ADD https://github.com/pgvector/pgvector.git#v0.8.1 /tmp/pgvector
|
||||
COPY . /tmp/pgvector
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-mark hold locales && \
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
||||
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
|
||||
Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
|
||||
|
||||
Portions Copyright (c) 1994, The Regents of the University of California
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.8.1",
|
||||
"version": "0.7.4",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -12,7 +12,7 @@
|
||||
"prereqs": {
|
||||
"runtime": {
|
||||
"requires": {
|
||||
"PostgreSQL": "13.0.0"
|
||||
"PostgreSQL": "12.0.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.8.1",
|
||||
"version": "0.7.4",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
9
Makefile
9
Makefile
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.8.1
|
||||
EXTVERSION = 0.7.4
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*--*.sql)
|
||||
@@ -76,9 +76,4 @@ docker:
|
||||
.PHONY: docker-release
|
||||
|
||||
docker-release:
|
||||
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm .
|
||||
|
||||
.PHONY: docker-release-trixie
|
||||
|
||||
docker-release-trixie:
|
||||
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie .
|
||||
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.8.1
|
||||
EXTVERSION = 0.7.4
|
||||
|
||||
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
|
||||
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
@@ -31,9 +31,6 @@ LIBDIR = $(PGROOT)\lib
|
||||
PKGLIBDIR = $(PGROOT)\lib
|
||||
SHAREDIR = $(PGROOT)\share
|
||||
|
||||
# Use $(PGROOT)\bin\pg_regress for Postgres < 17
|
||||
PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress
|
||||
|
||||
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
|
||||
|
||||
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
|
||||
@@ -57,11 +54,11 @@ install: all
|
||||
copy $(SHLIB) "$(PKGLIBDIR)"
|
||||
copy $(EXTENSION).control "$(SHAREDIR)\extension"
|
||||
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
|
||||
if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
|
||||
mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
|
||||
for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
|
||||
|
||||
installcheck:
|
||||
"$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
|
||||
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
|
||||
|
||||
uninstall:
|
||||
del /f "$(PKGLIBDIR)\$(SHLIB)"
|
||||
|
||||
257
README.md
257
README.md
@@ -17,11 +17,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
|
||||
|
||||
### Linux and Mac
|
||||
|
||||
Compile and install the extension (supports Postgres 13+)
|
||||
Compile and install the extension (supports Postgres 12+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -33,17 +33,27 @@ You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#p
|
||||
|
||||
### Windows
|
||||
|
||||
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build:
|
||||
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\18"
|
||||
call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
|
||||
```
|
||||
|
||||
Note: The exact path will vary depending on your Visual Studio version and edition
|
||||
|
||||
Then use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\16"
|
||||
cd %TEMP%
|
||||
git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
```
|
||||
|
||||
Note: Postgres 17 is not supported yet due to an upstream issue
|
||||
|
||||
See the [installation notes](#installation-notes---windows) if you run into issues
|
||||
|
||||
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
|
||||
@@ -74,7 +84,7 @@ Get the nearest neighbors by L2 distance
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`)
|
||||
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0)
|
||||
|
||||
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
|
||||
|
||||
@@ -138,9 +148,9 @@ Supported distance functions are:
|
||||
- `<->` - L2 distance
|
||||
- `<#>` - (negative) inner product
|
||||
- `<=>` - cosine distance
|
||||
- `<+>` - L1 distance
|
||||
- `<~>` - Hamming distance (binary vectors)
|
||||
- `<%>` - Jaccard distance (binary vectors)
|
||||
- `<+>` - L1 distance (added in 0.7.0)
|
||||
- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
|
||||
- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
|
||||
|
||||
Get the nearest neighbors to a row
|
||||
|
||||
@@ -227,19 +237,19 @@ Cosine distance
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
L1 distance
|
||||
L1 distance - added in 0.7.0
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
|
||||
```
|
||||
|
||||
Hamming distance
|
||||
Hamming distance - added in 0.7.0
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
|
||||
```
|
||||
|
||||
Jaccard distance
|
||||
Jaccard distance - added in 0.7.0
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
|
||||
@@ -248,9 +258,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
|
||||
Supported types are:
|
||||
|
||||
- `vector` - up to 2,000 dimensions
|
||||
- `halfvec` - up to 4,000 dimensions
|
||||
- `bit` - up to 64,000 dimensions
|
||||
- `sparsevec` - up to 1,000 non-zero elements
|
||||
- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
|
||||
- `bit` - up to 64,000 dimensions (added in 0.7.0)
|
||||
- `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0)
|
||||
|
||||
### Index Options
|
||||
|
||||
@@ -304,19 +314,17 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t
|
||||
|
||||
Like other index types, it’s faster to create an index after loading your initial data
|
||||
|
||||
You can also speed up index creation by increasing the number of parallel workers (2 by default)
|
||||
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
|
||||
|
||||
```sql
|
||||
SET max_parallel_maintenance_workers = 7; -- plus leader
|
||||
```
|
||||
|
||||
For a large number of workers, you may need to increase `max_parallel_workers` (8 by default)
|
||||
|
||||
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
|
||||
For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
|
||||
|
||||
### Indexing Progress
|
||||
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
|
||||
|
||||
```sql
|
||||
SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
|
||||
@@ -359,7 +367,7 @@ Cosine distance
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
Hamming distance
|
||||
Hamming distance - added in 0.7.0
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
|
||||
@@ -368,8 +376,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
|
||||
Supported types are:
|
||||
|
||||
- `vector` - up to 2,000 dimensions
|
||||
- `halfvec` - up to 4,000 dimensions
|
||||
- `bit` - up to 64,000 dimensions
|
||||
- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
|
||||
- `bit` - up to 64,000 dimensions (added in 0.7.0)
|
||||
|
||||
### Query Options
|
||||
|
||||
@@ -402,7 +410,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
|
||||
|
||||
### Indexing Progress
|
||||
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
|
||||
|
||||
```sql
|
||||
SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
|
||||
@@ -419,49 +427,25 @@ Note: `%` is only populated during the `loading tuples` phase
|
||||
|
||||
## Filtering
|
||||
|
||||
There are a few ways to index nearest neighbor queries with a `WHERE` clause.
|
||||
There are a few ways to index nearest neighbor queries with a `WHERE` clause
|
||||
|
||||
```sql
|
||||
SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
A good place to start is creating an index on the filter column. This can provide fast, exact nearest neighbor search in many cases. Postgres has a number of [index types](https://www.postgresql.org/docs/current/indexes-types.html) for this: B-tree (default), hash, GiST, SP-GiST, GIN, and BRIN.
|
||||
Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items (category_id);
|
||||
```
|
||||
|
||||
For multiple columns, consider a [multicolumn index](https://www.postgresql.org/docs/current/indexes-multicolumn.html).
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items (location_id, category_id);
|
||||
```
|
||||
|
||||
Exact indexes work well for conditions that match a low percentage of rows. Otherwise, [approximate indexes](#indexing) can work better.
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
|
||||
```
|
||||
|
||||
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, increase `hnsw.ef_search`.
|
||||
|
||||
```sql
|
||||
SET hnsw.ef_search = 200;
|
||||
```
|
||||
|
||||
Starting with 0.8.0, you can enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
|
||||
|
||||
```sql
|
||||
SET hnsw.iterative_scan = strict_order;
|
||||
```
|
||||
|
||||
If filtering by only a few distinct values, consider [partial indexing](https://www.postgresql.org/docs/current/indexes-partial.html).
|
||||
Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WHERE (category_id = 123);
|
||||
```
|
||||
|
||||
If filtering by many different values, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html).
|
||||
Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
|
||||
@@ -469,9 +453,11 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
|
||||
|
||||
## Iterative Index Scans
|
||||
|
||||
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
|
||||
*Unreleased*
|
||||
|
||||
Iterative scans can use strict or relaxed ordering.
|
||||
With approximate indexes, queries with filtering can return less results (due to post-filtering). Starting with 0.8.0, you can enable iterative index scans. If too few results from the initial scan match the filters, the scan will resume until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). This can significantly improve recall.
|
||||
|
||||
There are two modes for iterative scans: strict and relaxed.
|
||||
|
||||
Strict ensures results are in the exact order by distance
|
||||
|
||||
@@ -492,11 +478,9 @@ With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.o
|
||||
```sql
|
||||
WITH relaxed_results AS MATERIALIZED (
|
||||
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
|
||||
) SELECT * FROM relaxed_results ORDER BY distance + 0;
|
||||
) SELECT * FROM relaxed_results ORDER BY distance;
|
||||
```
|
||||
|
||||
Note: `+ 0` is needed for Postgres 17+
|
||||
|
||||
For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
|
||||
|
||||
```sql
|
||||
@@ -509,7 +493,7 @@ Note: Place any other filters inside the CTE
|
||||
|
||||
### Iterative Scan Options
|
||||
|
||||
Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends.
|
||||
Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends
|
||||
|
||||
#### HNSW
|
||||
|
||||
@@ -527,7 +511,18 @@ Specify the max amount of memory to use, as a multiple of `work_mem` (1 by defau
|
||||
SET hnsw.scan_mem_multiplier = 2;
|
||||
```
|
||||
|
||||
Note: Try increasing this if increasing `hnsw.max_scan_tuples` does not improve recall
|
||||
You can see when increasing this is needed by enabling debug messages
|
||||
|
||||
```sql
|
||||
SET client_min_messages = debug1;
|
||||
```
|
||||
|
||||
which will show when a scan reaches the memory limit
|
||||
|
||||
```text
|
||||
DEBUG: hnsw index scan reached memory limit after 20000 tuples
|
||||
HINT: Increase hnsw.scan_mem_multiplier to scan more tuples.
|
||||
```
|
||||
|
||||
#### IVFFlat
|
||||
|
||||
@@ -541,6 +536,8 @@ Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used
|
||||
|
||||
## Half-Precision Vectors
|
||||
|
||||
*Added in 0.7.0*
|
||||
|
||||
Use the `halfvec` type to store half-precision vectors
|
||||
|
||||
```sql
|
||||
@@ -549,6 +546,8 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
|
||||
|
||||
## Half-Precision Indexing
|
||||
|
||||
*Added in 0.7.0*
|
||||
|
||||
Index vectors at half precision for smaller indexes
|
||||
|
||||
```sql
|
||||
@@ -570,16 +569,24 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
|
||||
INSERT INTO items (embedding) VALUES ('000'), ('111');
|
||||
```
|
||||
|
||||
Get the nearest neighbors by Hamming distance
|
||||
Get the nearest neighbors by Hamming distance (added in 0.7.0)
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
|
||||
```
|
||||
|
||||
Or (before 0.7.0)
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
|
||||
```
|
||||
|
||||
Also supports Jaccard distance (`<%>`)
|
||||
|
||||
## Binary Quantization
|
||||
|
||||
*Added in 0.7.0*
|
||||
|
||||
Use expression indexing for binary quantization
|
||||
|
||||
```sql
|
||||
@@ -602,6 +609,8 @@ SELECT * FROM (
|
||||
|
||||
## Sparse Vectors
|
||||
|
||||
*Added in 0.7.0*
|
||||
|
||||
Use the `sparsevec` type to store sparse vectors
|
||||
|
||||
```sql
|
||||
@@ -635,6 +644,8 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
|
||||
|
||||
## Indexing Subvectors
|
||||
|
||||
*Added in 0.7.0*
|
||||
|
||||
Use expression indexing to index subvectors
|
||||
|
||||
```sql
|
||||
@@ -695,10 +706,10 @@ CREATE INDEX CONCURRENTLY ...
|
||||
|
||||
### Querying
|
||||
|
||||
Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance.
|
||||
Use `EXPLAIN ANALYZE` to debug performance.
|
||||
|
||||
```sql
|
||||
EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
#### Exact Search
|
||||
@@ -748,6 +759,8 @@ SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_t
|
||||
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
|
||||
```
|
||||
|
||||
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
|
||||
|
||||
Monitor recall by comparing results from approximate search with exact search.
|
||||
|
||||
```sql
|
||||
@@ -771,32 +784,25 @@ Use pgvector from any language with a Postgres client. You can even generate and
|
||||
|
||||
Language | Libraries / Examples
|
||||
--- | ---
|
||||
Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada)
|
||||
C | [pgvector-c](https://github.com/pgvector/pgvector-c)
|
||||
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
|
||||
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
|
||||
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
|
||||
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
|
||||
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
|
||||
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
|
||||
Erlang | [pgvector-erlang](https://github.com/pgvector/pgvector-erlang)
|
||||
Fortran | [pgvector-fortran](https://github.com/pgvector/pgvector-fortran)
|
||||
Gleam | [pgvector-gleam](https://github.com/pgvector/pgvector-gleam)
|
||||
Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
|
||||
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
|
||||
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
|
||||
JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
|
||||
Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl)
|
||||
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
|
||||
Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
|
||||
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
|
||||
Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
|
||||
OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml)
|
||||
Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal)
|
||||
Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
|
||||
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
|
||||
Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
|
||||
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
|
||||
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
|
||||
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
|
||||
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
|
||||
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
|
||||
@@ -814,11 +820,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
|
||||
|
||||
#### What if I want to index vectors with more than 2,000 dimensions?
|
||||
|
||||
You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
|
||||
You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
|
||||
|
||||
#### Can I store vectors with different dimensions in the same column?
|
||||
|
||||
You can use `vector` as the type (instead of `vector(n)`).
|
||||
You can use `vector` as the type (instead of `vector(3)`).
|
||||
|
||||
```sql
|
||||
CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
|
||||
@@ -918,7 +924,7 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
|
||||
|
||||
#### Why are there less results for a query after adding an HNSW index?
|
||||
|
||||
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`), which is 40 by default. There may be even less results due to dead tuples or filtering conditions in the query. Enabling [iterative index scans](#iterative-index-scans) can help address this.
|
||||
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
|
||||
|
||||
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
|
||||
|
||||
@@ -930,7 +936,7 @@ The index was likely created with too little data for the number of lists. Drop
|
||||
DROP INDEX index_name;
|
||||
```
|
||||
|
||||
Results can also be limited by the number of probes (`ivfflat.probes`). Enabling [iterative index scans](#iterative-index-scans) can address this.
|
||||
Results can also be limited by the number of probes (`ivfflat.probes`).
|
||||
|
||||
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
|
||||
|
||||
@@ -1066,7 +1072,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
|
||||
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
|
||||
|
||||
```sh
|
||||
export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config
|
||||
export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
|
||||
```
|
||||
|
||||
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
|
||||
@@ -1077,11 +1083,11 @@ sudo --preserve-env=PG_CONFIG make install
|
||||
|
||||
A few common paths on Mac are:
|
||||
|
||||
- EDB installer - `/Library/PostgreSQL/18/bin/pg_config`
|
||||
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config`
|
||||
- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config`
|
||||
- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
|
||||
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
|
||||
- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
|
||||
|
||||
Note: Replace `18` with your Postgres server version
|
||||
Note: Replace `17` with your Postgres server version
|
||||
|
||||
### Missing Header
|
||||
|
||||
@@ -1090,20 +1096,14 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
|
||||
For Ubuntu and Debian, use:
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql-server-dev-18
|
||||
sudo apt install postgresql-server-dev-17
|
||||
```
|
||||
|
||||
Note: Replace `18` with your Postgres server version
|
||||
Note: Replace `17` with your Postgres server version
|
||||
|
||||
### Missing SDK
|
||||
|
||||
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists.
|
||||
|
||||
```sh
|
||||
pg_config --cppflags
|
||||
```
|
||||
|
||||
Reinstall Postgres to fix this.
|
||||
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
|
||||
|
||||
### Portability
|
||||
|
||||
@@ -1121,14 +1121,6 @@ make OPTFLAGS=""
|
||||
|
||||
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
|
||||
|
||||
### Mismatched Architecture
|
||||
|
||||
If compilation fails with `error C2196: case value '4' already used`, make sure you’re using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
|
||||
|
||||
### Missing Symbol
|
||||
|
||||
If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+.
|
||||
|
||||
### Permissions
|
||||
|
||||
If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
|
||||
@@ -1140,38 +1132,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
|
||||
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
|
||||
|
||||
```sh
|
||||
docker pull pgvector/pgvector:pg18-trixie
|
||||
docker pull pgvector/pgvector:pg17
|
||||
```
|
||||
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way).
|
||||
|
||||
Supported tags are:
|
||||
|
||||
- `pg18-trixie`, `0.8.1-pg18-trixie`
|
||||
- `pg18-bookworm`, `0.8.1-pg18-bookworm`, `pg18`, `0.8.1-pg18`
|
||||
- `pg17-trixie`, `0.8.1-pg17-trixie`
|
||||
- `pg17-bookworm`, `0.8.1-pg17-bookworm`, `pg17`, `0.8.1-pg17`
|
||||
- `pg16-trixie`, `0.8.1-pg16-trixie`
|
||||
- `pg16-bookworm`, `0.8.1-pg16-bookworm`, `pg16`, `0.8.1-pg16`
|
||||
- `pg15-trixie`, `0.8.1-pg15-trixie`
|
||||
- `pg15-bookworm`, `0.8.1-pg15-bookworm`, `pg15`, `0.8.1-pg15`
|
||||
- `pg14-trixie`, `0.8.1-pg14-trixie`
|
||||
- `pg14-bookworm`, `0.8.1-pg14-bookworm`, `pg14`, `0.8.1-pg14`
|
||||
- `pg13-trixie`, `0.8.1-pg13-trixie`
|
||||
- `pg13-bookworm`, `0.8.1-pg13-bookworm`, `pg13`, `0.8.1-pg13`
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
|
||||
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
|
||||
|
||||
```sh
|
||||
docker run --shm-size=1g ...
|
||||
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
### Homebrew
|
||||
@@ -1182,7 +1153,7 @@ With Homebrew Postgres, you can use:
|
||||
brew install pgvector
|
||||
```
|
||||
|
||||
Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas
|
||||
Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
|
||||
|
||||
### PGXN
|
||||
|
||||
@@ -1197,29 +1168,29 @@ pgxn install vector
|
||||
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql-18-pgvector
|
||||
sudo apt install postgresql-17-pgvector
|
||||
```
|
||||
|
||||
Note: Replace `18` with your Postgres server version
|
||||
Note: Replace `17` with your Postgres server version
|
||||
|
||||
### Yum
|
||||
|
||||
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
|
||||
|
||||
```sh
|
||||
sudo yum install pgvector_18
|
||||
sudo yum install pgvector_17
|
||||
# or
|
||||
sudo dnf install pgvector_18
|
||||
sudo dnf install pgvector_17
|
||||
```
|
||||
|
||||
Note: Replace `18` with your Postgres server version
|
||||
Note: Replace `17` with your Postgres server version
|
||||
|
||||
### pkg
|
||||
|
||||
Install the FreeBSD package with:
|
||||
|
||||
```sh
|
||||
pkg install postgresql17-pgvector
|
||||
pkg install postgresql15-pgvector
|
||||
```
|
||||
|
||||
or the port with:
|
||||
@@ -1261,6 +1232,36 @@ You can check the version in the current database with:
|
||||
SELECT extversion FROM pg_extension WHERE extname = 'vector';
|
||||
```
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
### 0.6.0
|
||||
|
||||
#### Postgres 12
|
||||
|
||||
If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
|
||||
|
||||
```sql
|
||||
ALTER TYPE vector SET (STORAGE = external);
|
||||
```
|
||||
|
||||
Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
|
||||
|
||||
#### Docker
|
||||
|
||||
The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
|
||||
|
||||
```sh
|
||||
docker pull pgvector/pgvector:pg16
|
||||
# or
|
||||
docker pull pgvector/pgvector:0.6.0-pg16
|
||||
```
|
||||
|
||||
Also, if you’ve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
|
||||
|
||||
```sh
|
||||
docker run --shm-size=1g ...
|
||||
```
|
||||
|
||||
## Thanks
|
||||
|
||||
Thanks to:
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit
|
||||
@@ -898,21 +898,8 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
|
||||
half *ax = a->x;
|
||||
VarBit *result = InitBitVector(a->dim);
|
||||
unsigned char *rx = VARBITS(result);
|
||||
int i = 0;
|
||||
int count = (a->dim / 8) * 8;
|
||||
|
||||
/* Auto-vectorized on aarch64 */
|
||||
for (; i < count; i += 8)
|
||||
{
|
||||
unsigned char result_byte = 0;
|
||||
|
||||
for (int j = 0; j < 8; j++)
|
||||
result_byte |= (HalfToFloat4(ax[i + j]) > 0) << (7 - j);
|
||||
|
||||
rx[i / 8] = result_byte;
|
||||
}
|
||||
|
||||
for (; i < a->dim; i++)
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
rx[i / 8] |= (HalfToFloat4(ax[i]) > 0) << (7 - (i % 8));
|
||||
|
||||
PG_RETURN_VARBIT_P(result);
|
||||
|
||||
31
src/hnsw.c
31
src/hnsw.c
@@ -52,20 +52,12 @@ HnswInitLockTranche(void)
|
||||
sizeof(int) * 1,
|
||||
&found);
|
||||
if (!found)
|
||||
{
|
||||
#if PG_VERSION_NUM >= 190000
|
||||
tranche_ids[0] = LWLockNewTrancheId("HnswBuild");
|
||||
#else
|
||||
tranche_ids[0] = LWLockNewTrancheId();
|
||||
#endif
|
||||
}
|
||||
hnsw_lock_tranche_id = tranche_ids[0];
|
||||
LWLockRelease(AddinShmemInitLock);
|
||||
|
||||
#if PG_VERSION_NUM < 190000
|
||||
/* Per-backend registration of the tranche ID */
|
||||
LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -85,21 +77,21 @@ HnswInit(void)
|
||||
|
||||
DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
|
||||
"Valid range is 1..1000.", &hnsw_ef_search,
|
||||
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
DefineCustomEnumVariable("hnsw.iterative_scan", "Sets the mode for iterative scans",
|
||||
NULL, &hnsw_iterative_scan,
|
||||
HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
/* This is approximate and does not affect the initial scan */
|
||||
DefineCustomIntVariable("hnsw.max_scan_tuples", "Sets the max number of tuples to visit for iterative scans",
|
||||
NULL, &hnsw_max_scan_tuples,
|
||||
20000, 1, INT_MAX, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
20000, 1, INT_MAX, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
/* Same range as hash_mem_multiplier */
|
||||
DefineCustomRealVariable("hnsw.scan_mem_multiplier", "Sets the multiple of work_mem to use for iterative scans",
|
||||
NULL, &hnsw_scan_mem_multiplier,
|
||||
1, 1, 1000, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
1, 1, 1000, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
MarkGUCPrefixReserved("hnsw");
|
||||
}
|
||||
@@ -138,7 +130,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
Relation index;
|
||||
|
||||
/* Never use index without order */
|
||||
if (path->indexorderbys == NIL)
|
||||
if (path->indexorderbys == NULL)
|
||||
{
|
||||
*indexStartupCost = get_float8_infinity();
|
||||
*indexTotalCost = get_float8_infinity();
|
||||
@@ -267,11 +259,6 @@ hnswhandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amoptsprocnum = 0;
|
||||
amroutine->amcanorder = false;
|
||||
amroutine->amcanorderbyop = true;
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
amroutine->amcanhash = false;
|
||||
amroutine->amconsistentequality = false;
|
||||
amroutine->amconsistentordering = false;
|
||||
#endif
|
||||
amroutine->amcanbackward = false; /* can change direction mid-scan */
|
||||
amroutine->amcanunique = false;
|
||||
amroutine->amcanmulticol = false;
|
||||
@@ -304,9 +291,6 @@ hnswhandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amvacuumcleanup = hnswvacuumcleanup;
|
||||
amroutine->amcanreturn = NULL;
|
||||
amroutine->amcostestimate = hnswcostestimate;
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
amroutine->amgettreeheight = NULL;
|
||||
#endif
|
||||
amroutine->amoptions = hnswoptions;
|
||||
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
|
||||
amroutine->ambuildphasename = hnswbuildphasename;
|
||||
@@ -327,10 +311,5 @@ hnswhandler(PG_FUNCTION_ARGS)
|
||||
amroutine->aminitparallelscan = NULL;
|
||||
amroutine->amparallelrescan = NULL;
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
amroutine->amtranslatestrategy = NULL;
|
||||
amroutine->amtranslatecmptype = NULL;
|
||||
#endif
|
||||
|
||||
PG_RETURN_POINTER(amroutine);
|
||||
}
|
||||
|
||||
13
src/hnsw.h
13
src/hnsw.h
@@ -126,7 +126,7 @@ typedef struct HnswNeighborArray HnswNeighborArray;
|
||||
|
||||
#define HnswPtrDeclare(type, relptrtype, ptrtype) \
|
||||
relptr_declare(type, relptrtype); \
|
||||
typedef union { type *ptr; relptrtype relptr; } ptrtype
|
||||
typedef union { type *ptr; relptrtype relptr; } ptrtype;
|
||||
|
||||
/* Pointers that can be absolute or relative */
|
||||
/* Use char for DatumPtr so works with Pointer */
|
||||
@@ -362,13 +362,6 @@ typedef union
|
||||
ItemPointerData indextid;
|
||||
} HnswUnvisited;
|
||||
|
||||
typedef struct HnswReadStreamData
|
||||
{
|
||||
HnswUnvisited *unvisited;
|
||||
int unvisitedLength;
|
||||
int visited;
|
||||
} HnswReadStreamData;
|
||||
|
||||
typedef struct HnswScanOpaqueData
|
||||
{
|
||||
const HnswTypeInfo *typeInfo;
|
||||
@@ -424,13 +417,13 @@ bool HnswCheckNorm(HnswSupport * support, Datum value);
|
||||
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
||||
void HnswInitPage(Buffer buf, Page page);
|
||||
void HnswInit(void);
|
||||
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples, bool maintenance);
|
||||
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
|
||||
HnswElement HnswGetEntryPoint(Relation index);
|
||||
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
|
||||
void *HnswAlloc(HnswAllocator * allocator, Size size);
|
||||
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
|
||||
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
|
||||
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing, bool maintenance);
|
||||
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
|
||||
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec);
|
||||
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
|
||||
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
|
||||
|
||||
@@ -54,10 +54,6 @@
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memutils.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 140000
|
||||
#include "utils/backend_progress.h"
|
||||
#else
|
||||
@@ -402,7 +398,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
|
||||
* Update graph in memory
|
||||
*/
|
||||
static void
|
||||
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate)
|
||||
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
|
||||
{
|
||||
HnswGraph *graph = buildstate->graph;
|
||||
char *base = buildstate->hnswarea;
|
||||
@@ -461,10 +457,10 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
|
||||
}
|
||||
|
||||
/* Find neighbors for element */
|
||||
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false, true);
|
||||
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
|
||||
|
||||
/* Update graph in memory */
|
||||
UpdateGraphInMemory(support, element, m, entryPoint, buildstate);
|
||||
UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate);
|
||||
|
||||
/* Release entry lock */
|
||||
LWLockRelease(entryLock);
|
||||
@@ -1058,7 +1054,7 @@ ComputeParallelWorkers(Relation heap, Relation index)
|
||||
* Build graph
|
||||
*/
|
||||
static void
|
||||
BuildGraph(HnswBuildState * buildstate)
|
||||
BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
|
||||
{
|
||||
int parallel_workers = 0;
|
||||
|
||||
@@ -1106,7 +1102,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
|
||||
|
||||
InitBuildState(buildstate, heap, index, indexInfo, forkNum);
|
||||
|
||||
BuildGraph(buildstate);
|
||||
BuildGraph(buildstate, forkNum);
|
||||
|
||||
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
|
||||
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
|
||||
|
||||
@@ -9,10 +9,6 @@
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memutils.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Get the insert page
|
||||
*/
|
||||
@@ -664,7 +660,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
|
||||
* Update graph on disk
|
||||
*/
|
||||
static void
|
||||
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building)
|
||||
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
|
||||
{
|
||||
BlockNumber newInsertPage = InvalidBlockNumber;
|
||||
|
||||
@@ -729,10 +725,10 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
|
||||
}
|
||||
|
||||
/* Find neighbors for element */
|
||||
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false, building);
|
||||
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
|
||||
|
||||
/* Update graph on disk */
|
||||
UpdateGraphOnDisk(index, support, element, m, entryPoint, building);
|
||||
UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building);
|
||||
|
||||
/* Release lock */
|
||||
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);
|
||||
|
||||
@@ -37,11 +37,11 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
|
||||
for (int lc = entryPoint->level; lc >= 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL, false);
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples, false);
|
||||
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -72,7 +72,7 @@ ResumeScanItems(IndexScanDesc scan)
|
||||
ep = lappend(ep, sc);
|
||||
}
|
||||
|
||||
return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples, false);
|
||||
return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -193,10 +193,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
|
||||
/* Count index scan for stats */
|
||||
pgstat_count_index_scan(scan->indexRelation);
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
if (scan->instrument)
|
||||
scan->instrument->nsearches++;
|
||||
#endif
|
||||
|
||||
/* Safety check */
|
||||
if (scan->orderByData == NULL)
|
||||
@@ -244,8 +240,8 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
if (so->discarded == NULL)
|
||||
break;
|
||||
|
||||
/* Reached max number of tuples or memory limit */
|
||||
if (so->tuples >= hnsw_max_scan_tuples || MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
|
||||
/* Reached max number of tuples */
|
||||
if (so->tuples >= hnsw_max_scan_tuples)
|
||||
{
|
||||
if (pairingheap_is_empty(so->discarded))
|
||||
break;
|
||||
@@ -253,6 +249,21 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
/* Return remaining tuples */
|
||||
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
|
||||
}
|
||||
/* Prevent scans from consuming too much memory */
|
||||
else if (MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
|
||||
{
|
||||
if (pairingheap_is_empty(so->discarded))
|
||||
{
|
||||
ereport(DEBUG1,
|
||||
(errmsg("hnsw index scan reached memory limit after " INT64_FORMAT " tuples", so->tuples),
|
||||
errhint("Increase hnsw.scan_mem_multiplier to scan more tuples.")));
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
/* Return remaining tuples */
|
||||
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
|
||||
}
|
||||
else
|
||||
{
|
||||
/*
|
||||
|
||||
120
src/hnswutils.c
120
src/hnswutils.c
@@ -15,14 +15,6 @@
|
||||
#include "utils/memdebug.h"
|
||||
#include "utils/rel.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 170000
|
||||
#include "storage/read_stream.h"
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM < 170000
|
||||
static inline uint64
|
||||
murmurhash64(uint64 data)
|
||||
@@ -533,12 +525,14 @@ HnswGetDistance(Datum a, Datum b, HnswSupport * support)
|
||||
* Load an element and optionally get its distance from q
|
||||
*/
|
||||
static void
|
||||
HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance, HnswElement * element)
|
||||
HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance, HnswElement * element)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
HnswElementTuple etup;
|
||||
|
||||
/* Read vector */
|
||||
buf = ReadBuffer(index, blkno);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
|
||||
@@ -559,7 +553,7 @@ HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery
|
||||
if (distance == NULL || maxDistance == NULL || *distance < *maxDistance)
|
||||
{
|
||||
if (*element == NULL)
|
||||
*element = HnswInitElementFromBlock(BufferGetBlockNumber(buf), offno);
|
||||
*element = HnswInitElementFromBlock(blkno, offno);
|
||||
|
||||
HnswLoadElementFromTuple(*element, etup, true, loadVec);
|
||||
}
|
||||
@@ -573,9 +567,7 @@ HnswLoadElementImpl(Buffer buf, OffsetNumber offno, double *distance, HnswQuery
|
||||
void
|
||||
HnswLoadElement(HnswElement element, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance)
|
||||
{
|
||||
Buffer buf = ReadBuffer(index, element->blkno);
|
||||
|
||||
HnswLoadElementImpl(buf, element->offno, distance, q, index, support, loadVec, maxDistance, &element);
|
||||
HnswLoadElementImpl(element->blkno, element->offno, distance, q, index, support, loadVec, maxDistance, &element);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -815,31 +807,11 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
|
||||
}
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM >= 170000
|
||||
/*
|
||||
* Get next block number for read stream
|
||||
*/
|
||||
static BlockNumber
|
||||
HnswReadStreamNextBlock(ReadStream *stream, void *callback_private_data, void *per_buffer_data)
|
||||
{
|
||||
HnswReadStreamData *streamData = callback_private_data;
|
||||
OffsetNumber *offno = per_buffer_data;
|
||||
HnswUnvisited *uv;
|
||||
|
||||
if (streamData->visited == streamData->unvisitedLength)
|
||||
return InvalidBlockNumber;
|
||||
|
||||
uv = &streamData->unvisited[streamData->visited++];
|
||||
*offno = ItemPointerGetOffsetNumber(&uv->indextid);
|
||||
return ItemPointerGetBlockNumber(&uv->indextid);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Algorithm 2 from paper
|
||||
*/
|
||||
List *
|
||||
HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples, bool maintenance)
|
||||
HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples)
|
||||
{
|
||||
List *w = NIL;
|
||||
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
|
||||
@@ -854,27 +826,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
|
||||
int unvisitedLength;
|
||||
bool inMemory = index == NULL;
|
||||
|
||||
#if PG_VERSION_NUM >= 170000
|
||||
HnswReadStreamData streamData;
|
||||
ReadStream *stream = NULL;
|
||||
|
||||
if (!inMemory)
|
||||
{
|
||||
int flags = READ_STREAM_DEFAULT;
|
||||
|
||||
if (maintenance)
|
||||
{
|
||||
flags |= READ_STREAM_MAINTENANCE;
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
flags |= READ_STREAM_USE_BATCHING;
|
||||
#endif
|
||||
|
||||
stream = read_stream_begin_relation(flags, NULL, index, MAIN_FORKNUM, HnswReadStreamNextBlock, &streamData, sizeof(OffsetNumber));
|
||||
}
|
||||
#endif
|
||||
|
||||
if (v == NULL)
|
||||
{
|
||||
v = &vh;
|
||||
@@ -937,23 +888,13 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
|
||||
if (inMemory)
|
||||
HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize);
|
||||
else
|
||||
{
|
||||
HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
|
||||
|
||||
#if PG_VERSION_NUM >= 170000
|
||||
read_stream_resume(stream);
|
||||
|
||||
streamData.unvisited = unvisited;
|
||||
streamData.unvisitedLength = unvisitedLength;
|
||||
streamData.visited = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* OK to count elements instead of tuples */
|
||||
if (tuples != NULL)
|
||||
(*tuples) += unvisitedLength;
|
||||
|
||||
for (int i = 0;; i++)
|
||||
for (int i = 0; i < unvisitedLength; i++)
|
||||
{
|
||||
HnswElement eElement;
|
||||
HnswSearchCandidate *e;
|
||||
@@ -964,40 +905,18 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
|
||||
|
||||
if (inMemory)
|
||||
{
|
||||
if (i == unvisitedLength)
|
||||
break;
|
||||
|
||||
eElement = unvisited[i].element;
|
||||
eDistance = GetElementDistance(base, eElement, q, support);
|
||||
}
|
||||
else
|
||||
{
|
||||
Buffer buf;
|
||||
OffsetNumber offno;
|
||||
|
||||
#if PG_VERSION_NUM >= 170000
|
||||
void *offnoPtr;
|
||||
|
||||
buf = read_stream_next_buffer(stream, &offnoPtr);
|
||||
|
||||
if (!BufferIsValid(buf))
|
||||
break;
|
||||
|
||||
offno = *((OffsetNumber *) offnoPtr);
|
||||
#else
|
||||
ItemPointer indextid;
|
||||
|
||||
if (i == unvisitedLength)
|
||||
break;
|
||||
|
||||
indextid = &unvisited[i].indextid;
|
||||
buf = ReadBuffer(index, ItemPointerGetBlockNumber(indextid));
|
||||
offno = ItemPointerGetOffsetNumber(indextid);
|
||||
#endif
|
||||
ItemPointer indextid = &unvisited[i].indextid;
|
||||
BlockNumber blkno = ItemPointerGetBlockNumber(indextid);
|
||||
OffsetNumber offno = ItemPointerGetOffsetNumber(indextid);
|
||||
|
||||
/* Avoid any allocations if not adding */
|
||||
eElement = NULL;
|
||||
HnswLoadElementImpl(buf, offno, &eDistance, q, index, support, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
|
||||
HnswLoadElementImpl(blkno, offno, &eDistance, q, index, support, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
|
||||
|
||||
if (eElement == NULL)
|
||||
continue;
|
||||
@@ -1053,11 +972,6 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
|
||||
w = lappend(w, sc);
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM >= 170000
|
||||
if (!inMemory)
|
||||
read_stream_end(stream);
|
||||
#endif
|
||||
|
||||
return w;
|
||||
}
|
||||
|
||||
@@ -1353,7 +1267,7 @@ PrecomputeHash(char *base, HnswElement element)
|
||||
* Algorithm 1 from paper
|
||||
*/
|
||||
void
|
||||
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing, bool maintenance)
|
||||
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing)
|
||||
{
|
||||
List *ep;
|
||||
List *w;
|
||||
@@ -1380,7 +1294,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
/* 1st phase: greedy search to insert level */
|
||||
for (int lc = entryLevel; lc >= level + 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(base, &q, ep, 1, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL, maintenance);
|
||||
w = HnswSearchLayer(base, &q, ep, 1, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
@@ -1399,7 +1313,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
List *lw = NIL;
|
||||
ListCell *lc2;
|
||||
|
||||
w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL, maintenance);
|
||||
w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, support, m, true, skipElement, NULL, NULL, true, NULL);
|
||||
|
||||
/* Convert search candidates to candidates */
|
||||
foreach(lc2, w)
|
||||
@@ -1479,7 +1393,7 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
}
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
|
||||
Datum
|
||||
@@ -1492,7 +1406,7 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
}
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
|
||||
Datum
|
||||
@@ -1505,4 +1419,4 @@ hnsw_sparsevec_support(PG_FUNCTION_ARGS)
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -9,14 +9,6 @@
|
||||
#include "storage/lmgr.h"
|
||||
#include "utils/memutils.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
#define vacuum_delay_point() vacuum_delay_point(false)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Check if deleted list contains an index TID
|
||||
*/
|
||||
@@ -212,7 +204,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
|
||||
element->heaptidsLength = 0;
|
||||
|
||||
/* Find neighbors for element, skipping itself */
|
||||
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true, true);
|
||||
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true);
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
@@ -20,10 +20,6 @@
|
||||
#include "utils/memutils.h"
|
||||
#include "vector.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 140000
|
||||
#include "utils/backend_progress.h"
|
||||
#else
|
||||
@@ -142,7 +138,7 @@ SampleRows(IvfflatBuildState * buildstate)
|
||||
* Add tuple to sort
|
||||
*/
|
||||
static void
|
||||
AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
|
||||
AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
|
||||
{
|
||||
double distance;
|
||||
double minDistance = DBL_MAX;
|
||||
@@ -219,7 +215,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
|
||||
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
|
||||
|
||||
/* Add tuple to sort */
|
||||
AddTupleToSort(tid, values, buildstate);
|
||||
AddTupleToSort(index, tid, values, buildstate);
|
||||
|
||||
/* Reset memory context */
|
||||
MemoryContextSwitchTo(oldCtx);
|
||||
@@ -364,7 +360,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
|
||||
buildstate->sortdesc = CreateTemplateTupleDesc(3);
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", buildstate->tupdesc->attrs[0].atttypid, -1, 0);
|
||||
|
||||
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
|
||||
|
||||
@@ -474,8 +470,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum)
|
||||
* Create list pages
|
||||
*/
|
||||
static void
|
||||
CreateListPages(Relation index, VectorArray centers, int lists,
|
||||
ForkNumber forkNum, ListInfo * *listInfo)
|
||||
CreateListPages(Relation index, VectorArray centers, int dimensions,
|
||||
int lists, ForkNumber forkNum, ListInfo * *listInfo)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
@@ -1008,7 +1004,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
|
||||
|
||||
/* Create pages */
|
||||
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
|
||||
CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo);
|
||||
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
|
||||
CreateEntryPages(buildstate, forkNum);
|
||||
|
||||
/* Write WAL for initialization fork since GenericXLog functions do not */
|
||||
@@ -1027,10 +1023,6 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
|
||||
IndexBuildResult *result;
|
||||
IvfflatBuildState buildstate;
|
||||
|
||||
#ifdef IVFFLAT_BENCH
|
||||
SeedRandom(42);
|
||||
#endif
|
||||
|
||||
BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
|
||||
|
||||
result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
|
||||
|
||||
@@ -39,16 +39,16 @@ IvfflatInit(void)
|
||||
|
||||
DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
|
||||
"Valid range is 1..lists.", &ivfflat_probes,
|
||||
IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
DefineCustomEnumVariable("ivfflat.iterative_scan", "Sets the mode for iterative scans",
|
||||
NULL, &ivfflat_iterative_scan,
|
||||
IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
/* If this is less than probes, probes is used */
|
||||
DefineCustomIntVariable("ivfflat.max_probes", "Sets the max number of probes for iterative scans",
|
||||
NULL, &ivfflat_max_probes,
|
||||
IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
|
||||
|
||||
MarkGUCPrefixReserved("ivfflat");
|
||||
}
|
||||
@@ -92,7 +92,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
Relation index;
|
||||
|
||||
/* Never use index without order */
|
||||
if (path->indexorderbys == NIL)
|
||||
if (path->indexorderbys == NULL)
|
||||
{
|
||||
*indexStartupCost = get_float8_infinity();
|
||||
*indexTotalCost = get_float8_infinity();
|
||||
@@ -186,11 +186,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amoptsprocnum = 0;
|
||||
amroutine->amcanorder = false;
|
||||
amroutine->amcanorderbyop = true;
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
amroutine->amcanhash = false;
|
||||
amroutine->amconsistentequality = false;
|
||||
amroutine->amconsistentordering = false;
|
||||
#endif
|
||||
amroutine->amcanbackward = false; /* can change direction mid-scan */
|
||||
amroutine->amcanunique = false;
|
||||
amroutine->amcanmulticol = false;
|
||||
@@ -223,9 +218,6 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
|
||||
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
|
||||
amroutine->amcostestimate = ivfflatcostestimate;
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
amroutine->amgettreeheight = NULL;
|
||||
#endif
|
||||
amroutine->amoptions = ivfflatoptions;
|
||||
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
|
||||
amroutine->ambuildphasename = ivfflatbuildphasename;
|
||||
@@ -246,10 +238,5 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->aminitparallelscan = NULL;
|
||||
amroutine->amparallelrescan = NULL;
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
amroutine->amtranslatestrategy = NULL;
|
||||
amroutine->amtranslatecmptype = NULL;
|
||||
#endif
|
||||
|
||||
PG_RETURN_POINTER(amroutine);
|
||||
}
|
||||
|
||||
@@ -13,10 +13,6 @@
|
||||
#include "utils/tuplesort.h"
|
||||
#include "vector.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 150000
|
||||
#include "common/pg_prng.h"
|
||||
#endif
|
||||
@@ -77,11 +73,9 @@
|
||||
#if PG_VERSION_NUM >= 150000
|
||||
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
|
||||
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
|
||||
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
|
||||
#else
|
||||
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
|
||||
#define RandomInt() random()
|
||||
#define SeedRandom(seed) srandom(seed)
|
||||
#endif
|
||||
|
||||
/* Variables */
|
||||
|
||||
@@ -65,7 +65,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
|
||||
* Insert a tuple into the index
|
||||
*/
|
||||
static void
|
||||
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
|
||||
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
|
||||
{
|
||||
const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
|
||||
IndexTuple itup;
|
||||
@@ -204,7 +204,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
|
||||
oldCtx = MemoryContextSwitchTo(insertCtx);
|
||||
|
||||
/* Insert tuple */
|
||||
InsertTuple(index, values, isnull, heap_tid);
|
||||
InsertTuple(index, values, isnull, heap_tid, heap);
|
||||
|
||||
/* Delete memory context */
|
||||
MemoryContextSwitchTo(oldCtx);
|
||||
|
||||
@@ -13,10 +13,6 @@
|
||||
#include "utils/memutils.h"
|
||||
#include "vector.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Initialize with kmeans++
|
||||
*
|
||||
|
||||
@@ -114,6 +114,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
{
|
||||
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
|
||||
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
|
||||
double tuples = 0;
|
||||
TupleTableSlot *slot = so->vslot;
|
||||
int batchProbes = 0;
|
||||
|
||||
@@ -160,6 +161,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
ExecStoreVirtualTuple(slot);
|
||||
|
||||
tuplesort_puttupleslot(so->sortstate, slot);
|
||||
|
||||
tuples++;
|
||||
}
|
||||
|
||||
searchPage = IvfflatPageGetOpaque(page)->nextblkno;
|
||||
@@ -168,6 +171,12 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
}
|
||||
}
|
||||
|
||||
if (tuples < 100 && ivfflat_iterative_scan == IVFFLAT_ITERATIVE_SCAN_OFF)
|
||||
ereport(DEBUG1,
|
||||
(errmsg("index scan found few tuples"),
|
||||
errdetail("Index may have been created with little data."),
|
||||
errhint("Recreate the index and possibly decrease lists.")));
|
||||
|
||||
tuplesort_performsort(so->sortstate);
|
||||
|
||||
#if defined(IVFFLAT_MEMORY)
|
||||
@@ -355,10 +364,6 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
|
||||
/* Count index scan for stats */
|
||||
pgstat_count_index_scan(scan->indexRelation);
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
if (scan->instrument)
|
||||
scan->instrument->nsearches++;
|
||||
#endif
|
||||
|
||||
/* Safety check */
|
||||
if (scan->orderByData == NULL)
|
||||
|
||||
@@ -259,8 +259,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x)
|
||||
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
|
||||
vec->dim = dimensions;
|
||||
|
||||
for (int i = 0; i < dimensions; i++)
|
||||
vec->x[i] = x[i];
|
||||
for (int k = 0; k < dimensions; k++)
|
||||
vec->x[k] = x[k];
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -271,8 +271,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
|
||||
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
|
||||
vec->dim = dimensions;
|
||||
|
||||
for (int i = 0; i < dimensions; i++)
|
||||
vec->x[i] = Float4ToHalfUnchecked(x[i]);
|
||||
for (int k = 0; k < dimensions; k++)
|
||||
vec->x[k] = Float4ToHalfUnchecked(x[k]);
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -284,33 +284,29 @@ BitUpdateCenter(Pointer v, int dimensions, float *x)
|
||||
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
|
||||
VARBITLEN(vec) = dimensions;
|
||||
|
||||
for (uint32 i = 0; i < VARBITBYTES(vec); i++)
|
||||
nx[i] = 0;
|
||||
for (uint32 k = 0; k < VARBITBYTES(vec); k++)
|
||||
nx[k] = 0;
|
||||
|
||||
for (int i = 0; i < dimensions; i++)
|
||||
nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8));
|
||||
for (int k = 0; k < dimensions; k++)
|
||||
nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
|
||||
}
|
||||
|
||||
static void
|
||||
VectorSumCenter(Pointer v, float *x)
|
||||
{
|
||||
Vector *vec = (Vector *) v;
|
||||
int dim = vec->dim;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < dim; i++)
|
||||
x[i] += vec->x[i];
|
||||
for (int k = 0; k < vec->dim; k++)
|
||||
x[k] += vec->x[k];
|
||||
}
|
||||
|
||||
static void
|
||||
HalfvecSumCenter(Pointer v, float *x)
|
||||
{
|
||||
HalfVector *vec = (HalfVector *) v;
|
||||
int dim = vec->dim;
|
||||
|
||||
/* Auto-vectorized on aarch64 */
|
||||
for (int i = 0; i < dim; i++)
|
||||
x[i] += HalfToFloat4(vec->x[i]);
|
||||
for (int k = 0; k < vec->dim; k++)
|
||||
x[k] += HalfToFloat4(vec->x[k]);
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -318,8 +314,8 @@ BitSumCenter(Pointer v, float *x)
|
||||
{
|
||||
VarBit *vec = (VarBit *) v;
|
||||
|
||||
for (int i = 0; i < VARBITLEN(vec); i++)
|
||||
x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01);
|
||||
for (int k = 0; k < VARBITLEN(vec); k++)
|
||||
x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -359,7 +355,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
}
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
|
||||
Datum
|
||||
@@ -374,4 +370,4 @@ ivfflat_bit_support(PG_FUNCTION_ARGS)
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -5,10 +5,6 @@
|
||||
#include "ivfflat.h"
|
||||
#include "storage/bufmgr.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
#define vacuum_delay_point() vacuum_delay_point(false)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Bulk delete tuples from the index
|
||||
*/
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
#include <math.h>
|
||||
|
||||
#include "catalog/pg_type.h"
|
||||
#include "common/shortest_dec.h"
|
||||
#include "common/string.h"
|
||||
#include "fmgr.h"
|
||||
#include "halfutils.h"
|
||||
@@ -13,10 +12,17 @@
|
||||
#include "sparsevec.h"
|
||||
#include "utils/array.h"
|
||||
#include "utils/builtins.h"
|
||||
#include "utils/float.h"
|
||||
#include "utils/lsyscache.h"
|
||||
#include "vector.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
#include "common/shortest_dec.h"
|
||||
#include "utils/float.h"
|
||||
#else
|
||||
#include <float.h>
|
||||
#include "utils/builtins.h"
|
||||
#endif
|
||||
|
||||
typedef struct SparseInputElement
|
||||
{
|
||||
int32 index;
|
||||
|
||||
27
src/vector.c
27
src/vector.c
@@ -35,11 +35,7 @@
|
||||
#define VECTOR_TARGET_CLONES
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.1");
|
||||
#else
|
||||
PG_MODULE_MAGIC;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Initialize index options and variables
|
||||
@@ -924,13 +920,11 @@ vector_concat(PG_FUNCTION_ARGS)
|
||||
CheckDim(dim);
|
||||
result = InitVector(dim);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
result->x[i] = a->x[i];
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++)
|
||||
result->x[i + start] = b->x[i];
|
||||
for (int i = 0; i < b->dim; i++)
|
||||
result->x[i + a->dim] = b->x[i];
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
@@ -946,21 +940,8 @@ binary_quantize(PG_FUNCTION_ARGS)
|
||||
float *ax = a->x;
|
||||
VarBit *result = InitBitVector(a->dim);
|
||||
unsigned char *rx = VARBITS(result);
|
||||
int i = 0;
|
||||
int count = (a->dim / 8) * 8;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (; i < count; i += 8)
|
||||
{
|
||||
unsigned char result_byte = 0;
|
||||
|
||||
for (int j = 0; j < 8; j++)
|
||||
result_byte |= (ax[i + j] > 0) << (7 - j);
|
||||
|
||||
rx[i / 8] = result_byte;
|
||||
}
|
||||
|
||||
for (; i < a->dim; i++)
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
|
||||
|
||||
PG_RETURN_VARBIT_P(result);
|
||||
|
||||
@@ -540,12 +540,6 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec)
|
||||
01001110101
|
||||
(1 row)
|
||||
|
||||
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
|
||||
binary_quantize
|
||||
---------------------
|
||||
1110110110011011011
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
|
||||
subvector
|
||||
-----------
|
||||
|
||||
@@ -123,12 +123,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
[0,0,0]
|
||||
(3 rows)
|
||||
|
||||
TRUNCATE t;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
RESET hnsw.iterative_scan;
|
||||
RESET hnsw.ef_search;
|
||||
DROP TABLE t;
|
||||
@@ -196,6 +190,4 @@ SHOW hnsw.scan_mem_multiplier;
|
||||
|
||||
SET hnsw.scan_mem_multiplier = 0;
|
||||
ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
|
||||
SET hnsw.scan_mem_multiplier = 1001;
|
||||
ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -110,15 +110,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
[1,1,1]
|
||||
(2 rows)
|
||||
|
||||
TRUNCATE t;
|
||||
NOTICE: ivfflat index created with little data
|
||||
DETAIL: This will cause low recall.
|
||||
HINT: Drop the index until the table has more data.
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
RESET ivfflat.iterative_scan;
|
||||
RESET ivfflat.max_probes;
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -576,12 +576,6 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
|
||||
01001110101
|
||||
(1 row)
|
||||
|
||||
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
|
||||
binary_quantize
|
||||
---------------------
|
||||
1110110110011011011
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
|
||||
subvector
|
||||
-----------
|
||||
|
||||
@@ -121,7 +121,6 @@ SELECT l2_normalize('[65504]'::halfvec);
|
||||
|
||||
SELECT binary_quantize('[1,0,-1]'::halfvec);
|
||||
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
|
||||
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
|
||||
|
||||
@@ -70,9 +70,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
SET hnsw.iterative_scan = relaxed_order;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
|
||||
TRUNCATE t;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
|
||||
RESET hnsw.iterative_scan;
|
||||
RESET hnsw.ef_search;
|
||||
DROP TABLE t;
|
||||
@@ -112,6 +109,5 @@ SET hnsw.max_scan_tuples = 0;
|
||||
SHOW hnsw.scan_mem_multiplier;
|
||||
|
||||
SET hnsw.scan_mem_multiplier = 0;
|
||||
SET hnsw.scan_mem_multiplier = 1001;
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -59,9 +59,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
SET ivfflat.max_probes = 2;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
|
||||
TRUNCATE t;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
|
||||
RESET ivfflat.iterative_scan;
|
||||
RESET ivfflat.max_probes;
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -128,7 +128,6 @@ SELECT l2_normalize('[3e38]'::vector);
|
||||
|
||||
SELECT binary_quantize('[1,0,-1]'::vector);
|
||||
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
|
||||
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
|
||||
|
||||
@@ -56,4 +56,13 @@ foreach ((30000, 50000, 70000))
|
||||
cmp_ok($avg, '<', $expected + 2);
|
||||
}
|
||||
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.iterative_scan = relaxed_order;
|
||||
SET client_min_messages = debug1;
|
||||
SET work_mem = '1MB';
|
||||
SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
|
||||
));
|
||||
like($stderr, qr/hnsw index scan reached memory limit after \d+ tuples/);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat and hnsw access methods'
|
||||
default_version = '0.8.1'
|
||||
default_version = '0.7.4'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user