Compare commits

..

39 Commits

Author SHA1 Message Date
Andrew Kane
42c930231a Test with ARM on CI 2025-01-19 01:29:28 -08:00
Andrew Kane
2fe560dc58 Fixed extra-semi warnings [skip ci] 2025-01-18 13:08:11 -08:00
Andrew Kane
b46beada1a Restored assertion checking on CI for Postgres 18 [skip ci] 2025-01-15 19:24:24 -08:00
Andrew Kane
0a42bc7aa5 Fixed undefined symbol: verify_compact_attribute error with Postgres 18 on CI 2025-01-10 13:54:18 -08:00
Andrew Kane
f5df32c41d Updated license year [skip ci] 2025-01-10 08:51:07 -08:00
Andrew Kane
2c53c30415 Fixed compilation error with Postgres 18 2025-01-10 08:50:51 -08:00
Rui Chen
b4bc010459 chore(ci): update to use pg 17 (#755)
* chore(ci): update to use pg 17

Signed-off-by: Rui Chen <rui@chenrui.dev>

* chore(ci): update pg 14 to `REL_14_15`

Signed-off-by: Rui Chen <rui@chenrui.dev>

---------

Signed-off-by: Rui Chen <rui@chenrui.dev>
2025-01-10 08:39:32 -08:00
Andrew Kane
7b4ff9b59f Updated CI to support macos-15 [skip ci] 2025-01-10 08:37:52 -08:00
Andrew Kane
cfdcbd75d1 Updated FreeBSD docs [skip ci] 2024-12-09 08:11:11 -08:00
Andrew Kane
5136983f35 Added link to pgvector-fortran [skip ci] 2024-12-06 09:39:02 -08:00
Andrew Kane
4ab4b89980 Added link to pgvector-erlang [skip ci] 2024-12-06 06:16:34 -08:00
Andrew Kane
85f0e3ccf6 Added link to pgvector-gleam [skip ci] 2024-12-05 19:29:57 -08:00
Andrew Kane
28e797cb5a Added link to pgvector-d [skip ci] 2024-12-05 16:56:21 -08:00
Andrew Kane
1263d753be Added link to pgvector-raku [skip ci] 2024-12-05 07:13:39 -08:00
Andrew Kane
5bc7937715 Added iterative index scans to troubleshooting docs [skip ci] 2024-11-22 15:22:06 -08:00
Andrew Kane
e7e899e9af Updated readme [skip ci] 2024-11-22 11:55:46 -08:00
Andrew Kane
2627c5ff77 Version bump to 0.8.0 [skip ci] 2024-10-30 13:06:34 -07:00
Andrew Kane
34b3cfdc43 Updated min Postgres version in META.json [skip ci] 2024-10-30 13:06:12 -07:00
Andrew Kane
cd218aae5a Removed unneeded code 2024-10-30 13:05:10 -07:00
Andrew Kane
ba9367f86c Updated readme [skip ci] 2024-10-30 12:58:00 -07:00
Andrew Kane
9c20550a41 Updated readme 2024-10-30 12:54:42 -07:00
Andrew Kane
e3e74fe94e Updated readme [skip ci] 2024-10-29 00:04:35 -07:00
Andrew Kane
96a5a44632 Updated readme [skip ci] 2024-10-28 23:52:16 -07:00
Andrew Kane
67e1392a83 Updated readme [skip ci] 2024-10-28 22:58:56 -07:00
Andrew Kane
e530a1a026 Updated readme [skip ci] 2024-10-28 22:49:41 -07:00
Andrew Kane
6170e2645b Updated readme [skip ci] 2024-10-28 22:44:44 -07:00
Andrew Kane
e6bae175f1 Updated readme [skip ci] 2024-10-28 22:26:50 -07:00
Andrew Kane
52b777e04a Updated readme [skip ci] 2024-10-28 22:01:27 -07:00
Andrew Kane
307271214f Updated readme [skip ci] 2024-10-28 21:42:05 -07:00
Andrew Kane
6e9f74ddce Updated readme [skip ci] 2024-10-28 20:04:09 -07:00
Andrew Kane
258215ad97 Improved test [skip ci] 2024-10-28 15:54:39 -07:00
Andrew Kane
fb87b6da91 Fixed test 2024-10-28 13:55:26 -07:00
Andrew Kane
a2a0b377f0 Removed memory limit debug message from HNSW index scans (EXPLAIN ANALYZE can be used instead) 2024-10-28 13:46:49 -07:00
Andrew Kane
2f770307b8 Removed unused variable [skip ci] 2024-10-28 13:38:42 -07:00
Andrew Kane
c04e16ff5b Removed debug message from IVFFlat index scans [skip ci] 2024-10-28 13:36:57 -07:00
Andrew Kane
bd4d272f26 Updated changelog [skip ci] 2024-10-28 13:05:27 -07:00
Andrew Kane
8bb797cc2f Updated changelog [skip ci] 2024-10-28 13:00:55 -07:00
Andrew Kane
fe6ec03dac Improved filtering section [skip ci] 2024-10-28 12:08:27 -07:00
Andrew Kane
c1161f8889 Updated readme [skip ci] 2024-10-28 02:08:53 -07:00
20 changed files with 81 additions and 198 deletions

View File

@@ -8,18 +8,10 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 18
os: ubuntu-24.04
- postgres: 17
os: ubuntu-24.04
- postgres: 16
os: ubuntu-22.04
- postgres: 15
os: ubuntu-22.04
os: ubuntu-24.04-arm
- postgres: 14
os: ubuntu-20.04
- postgres: 13
os: ubuntu-20.04
os: ubuntu-22.04-arm
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
@@ -39,94 +31,3 @@ jobs:
sudo apt-get update
sudo apt-get install libipc-run-perl
- run: make prove_installcheck
mac:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 16
os: macos-14
- postgres: 14
os: macos-13
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
# Homebrew Postgres does not enable TAP tests, so need to download
- run: |
brew install cpanm
cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
env:
TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }}
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
- if: ${{ failure() }}
run: cat regression.diffs
i386:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
container:
image: debian:12
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- if: ${{ failure() }}
run: cat pgvector/regression.diffs
valgrind:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 16
check-ub: yes
- run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install
- run: make installcheck

View File

@@ -1,10 +1,10 @@
## 0.8.0 (unreleased)
## 0.8.0 (2024-10-30)
- Added support for iterative index scans
- Added casts for arrays to `sparsevec`
- Improved cost estimation
- Improved cost estimation for better index selection when filtering
- Improved performance of HNSW index scans
- Improved performance of HNSW inserts and on-disk index builds
- Reduced memory usage for HNSW index scans
- Dropped support for Postgres 12
## 0.7.4 (2024-08-05)

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.7.4",
"version": "0.8.0",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -12,7 +12,7 @@
"prereqs": {
"runtime": {
"requires": {
"PostgreSQL": "12.0.0"
"PostgreSQL": "13.0.0"
}
}
},
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.7.4",
"version": "0.8.0",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.7.4
EXTVERSION = 0.8.0
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.7.4
EXTVERSION = 0.8.0
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

View File

@@ -17,11 +17,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
### Linux and Mac
Compile and install the extension (supports Postgres 12+)
Compile and install the extension (supports Postgres 13+)
```sh
cd /tmp
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -46,13 +46,13 @@ Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP%
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
```
Note: Postgres 17 is not supported yet due to an upstream issue
Note: Postgres 17 is not supported with MSVC yet due to an [upstream issue](https://www.postgresql.org/message-id/flat/CAOdR5yF0krWrxycA04rgUKCgKugRvGWzzGLAhDZ9bzNv8g0Lag%40mail.gmail.com)
See the [installation notes](#installation-notes---windows) if you run into issues
@@ -324,7 +324,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
```sql
SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -410,7 +410,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
```sql
SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -427,25 +427,49 @@ Note: `%` is only populated during the `loading tuples` phase
## Filtering
There are a few ways to index nearest neighbor queries with a `WHERE` clause
There are a few ways to index nearest neighbor queries with a `WHERE` clause.
```sql
SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
A good place to start is creating an index on the filter column. This can provide fast, exact nearest neighbor search in many cases. Postgres has a number of [index types](https://www.postgresql.org/docs/current/indexes-types.html) for this: B-tree (default), hash, GiST, SP-GiST, GIN, and BRIN.
```sql
CREATE INDEX ON items (category_id);
```
Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
For multiple columns, consider a [multicolumn index](https://www.postgresql.org/docs/current/indexes-multicolumn.html).
```sql
CREATE INDEX ON items (location_id, category_id);
```
Exact indexes work well for conditions that match a low percentage of rows. Otherwise, [approximate indexes](#indexing) can work better.
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
```
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, increase `hnsw.ef_search`.
```sql
SET hnsw.ef_search = 200;
```
Starting with 0.8.0, you can enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
```sql
SET hnsw.iterative_scan = strict_order;
```
If filtering by only a few distinct values, consider [partial indexing](https://www.postgresql.org/docs/current/indexes-partial.html).
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WHERE (category_id = 123);
```
Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
If filtering by many different values, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html).
```sql
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
@@ -453,11 +477,11 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
## Iterative Index Scans
*Unreleased*
*Added in 0.8.0*
With approximate indexes, queries with filtering can return less results (due to post-filtering). Starting with 0.8.0, you can enable iterative index scans. If too few results from the initial scan match the filters, the scan will resume until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). This can significantly improve recall.
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
There are two modes for iterative scans: strict and relaxed.
Iterative scans can use strict or relaxed ordering.
Strict ensures results are in the exact order by distance
@@ -493,7 +517,7 @@ Note: Place any other filters inside the CTE
### Iterative Scan Options
Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends
Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends.
#### HNSW
@@ -511,18 +535,7 @@ Specify the max amount of memory to use, as a multiple of `work_mem` (1 by defau
SET hnsw.scan_mem_multiplier = 2;
```
You can see when increasing this is needed by enabling debug messages
```sql
SET client_min_messages = debug1;
```
which will show when a scan reaches the memory limit
```text
DEBUG: hnsw index scan reached memory limit after 20000 tuples
HINT: Increase hnsw.scan_mem_multiplier to scan more tuples.
```
Note: Try increasing this if increasing `hnsw.max_scan_tuples` does not improve recall
#### IVFFlat
@@ -788,8 +801,12 @@ C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
Erlang | [pgvector-erlang](https://github.com/pgvector/pgvector-erlang)
Fortran | [pgvector-fortran](https://github.com/pgvector/pgvector-fortran)
Gleam | [pgvector-gleam](https://github.com/pgvector/pgvector-gleam)
Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
@@ -803,6 +820,7 @@ Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
@@ -924,7 +942,7 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
#### Why are there less results for a query after adding an HNSW index?
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`), which is 40 by default. There may be even less results due to dead tuples or filtering conditions in the query. Enabling [iterative index scans](#iterative-index-scans) can help address this.
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -936,7 +954,7 @@ The index was likely created with too little data for the number of lists. Drop
DROP INDEX index_name;
```
Results can also be limited by the number of probes (`ivfflat.probes`).
Results can also be limited by the number of probes (`ivfflat.probes`). Enabling [iterative index scans](#iterative-index-scans) can address this.
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -1140,7 +1158,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually:
```sh
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
```
@@ -1190,7 +1208,7 @@ Note: Replace `17` with your Postgres server version
Install the FreeBSD package with:
```sh
pkg install postgresql15-pgvector
pkg install postgresql16-pgvector
```
or the port with:

View File

@@ -77,21 +77,21 @@ HnswInit(void)
DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
"Valid range is 1..1000.", &hnsw_ef_search,
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
DefineCustomEnumVariable("hnsw.iterative_scan", "Sets the mode for iterative scans",
NULL, &hnsw_iterative_scan,
HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
/* This is approximate and does not affect the initial scan */
DefineCustomIntVariable("hnsw.max_scan_tuples", "Sets the max number of tuples to visit for iterative scans",
NULL, &hnsw_max_scan_tuples,
20000, 1, INT_MAX, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
20000, 1, INT_MAX, PGC_USERSET, 0, NULL, NULL, NULL);
/* Same range as hash_mem_multiplier */
DefineCustomRealVariable("hnsw.scan_mem_multiplier", "Sets the multiple of work_mem to use for iterative scans",
NULL, &hnsw_scan_mem_multiplier,
1, 1, 1000, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
1, 1, 1000, PGC_USERSET, 0, NULL, NULL, NULL);
MarkGUCPrefixReserved("hnsw");
}

View File

@@ -126,7 +126,7 @@ typedef struct HnswNeighborArray HnswNeighborArray;
#define HnswPtrDeclare(type, relptrtype, ptrtype) \
relptr_declare(type, relptrtype); \
typedef union { type *ptr; relptrtype relptr; } ptrtype;
typedef union { type *ptr; relptrtype relptr; } ptrtype
/* Pointers that can be absolute or relative */
/* Use char for DatumPtr so works with Pointer */

View File

@@ -240,8 +240,8 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
if (so->discarded == NULL)
break;
/* Reached max number of tuples */
if (so->tuples >= hnsw_max_scan_tuples)
/* Reached max number of tuples or memory limit */
if (so->tuples >= hnsw_max_scan_tuples || MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
{
if (pairingheap_is_empty(so->discarded))
break;
@@ -249,21 +249,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
/* Return remaining tuples */
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
}
/* Prevent scans from consuming too much memory */
else if (MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
{
if (pairingheap_is_empty(so->discarded))
{
ereport(DEBUG1,
(errmsg("hnsw index scan reached memory limit after " INT64_FORMAT " tuples", so->tuples),
errhint("Increase hnsw.scan_mem_multiplier to scan more tuples.")));
break;
}
/* Return remaining tuples */
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
}
else
{
/*

View File

@@ -1393,7 +1393,7 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
};
PG_RETURN_POINTER(&typeInfo);
};
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
Datum
@@ -1406,7 +1406,7 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
};
PG_RETURN_POINTER(&typeInfo);
};
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
Datum
@@ -1419,4 +1419,4 @@ hnsw_sparsevec_support(PG_FUNCTION_ARGS)
};
PG_RETURN_POINTER(&typeInfo);
};
}

View File

@@ -360,7 +360,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->sortdesc = CreateTemplateTupleDesc(3);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", buildstate->tupdesc->attrs[0].atttypid, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);

View File

@@ -39,16 +39,16 @@ IvfflatInit(void)
DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
"Valid range is 1..lists.", &ivfflat_probes,
IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
DefineCustomEnumVariable("ivfflat.iterative_scan", "Sets the mode for iterative scans",
NULL, &ivfflat_iterative_scan,
IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
/* If this is less than probes, probes is used */
DefineCustomIntVariable("ivfflat.max_probes", "Sets the max number of probes for iterative scans",
NULL, &ivfflat_max_probes,
IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
MarkGUCPrefixReserved("ivfflat");
}

View File

@@ -114,7 +114,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
double tuples = 0;
TupleTableSlot *slot = so->vslot;
int batchProbes = 0;
@@ -161,8 +160,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
ExecStoreVirtualTuple(slot);
tuplesort_puttupleslot(so->sortstate, slot);
tuples++;
}
searchPage = IvfflatPageGetOpaque(page)->nextblkno;
@@ -171,12 +168,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
}
}
if (tuples < 100 && ivfflat_iterative_scan == IVFFLAT_ITERATIVE_SCAN_OFF)
ereport(DEBUG1,
(errmsg("index scan found few tuples"),
errdetail("Index may have been created with little data."),
errhint("Recreate the index and possibly decrease lists.")));
tuplesort_performsort(so->sortstate);
#if defined(IVFFLAT_MEMORY)

View File

@@ -355,7 +355,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
};
PG_RETURN_POINTER(&typeInfo);
};
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
Datum
@@ -370,4 +370,4 @@ ivfflat_bit_support(PG_FUNCTION_ARGS)
};
PG_RETURN_POINTER(&typeInfo);
};
}

View File

@@ -4,6 +4,7 @@
#include <math.h>
#include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "common/string.h"
#include "fmgr.h"
#include "halfutils.h"
@@ -12,17 +13,10 @@
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/lsyscache.h"
#include "vector.h"
#if PG_VERSION_NUM >= 120000
#include "common/shortest_dec.h"
#include "utils/float.h"
#else
#include <float.h>
#include "utils/builtins.h"
#endif
typedef struct SparseInputElement
{
int32 index;

View File

@@ -190,4 +190,6 @@ SHOW hnsw.scan_mem_multiplier;
SET hnsw.scan_mem_multiplier = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
SET hnsw.scan_mem_multiplier = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
DROP TABLE t;

View File

@@ -109,5 +109,6 @@ SET hnsw.max_scan_tuples = 0;
SHOW hnsw.scan_mem_multiplier;
SET hnsw.scan_mem_multiplier = 0;
SET hnsw.scan_mem_multiplier = 1001;
DROP TABLE t;

View File

@@ -56,13 +56,4 @@ foreach ((30000, 50000, 70000))
cmp_ok($avg, '<', $expected + 2);
}
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET enable_seqscan = off;
SET hnsw.iterative_scan = relaxed_order;
SET client_min_messages = debug1;
SET work_mem = '1MB';
SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
));
like($stderr, qr/hnsw index scan reached memory limit after \d+ tuples/);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.7.4'
default_version = '0.8.0'
module_pathname = '$libdir/vector'
relocatable = true