Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
ec68b75892 Added bulk hashing [skip ci] 2024-10-29 11:41:57 -07:00
20 changed files with 128 additions and 111 deletions

View File

@@ -46,8 +46,8 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- postgres: 17 - postgres: 16
os: macos-15 os: macos-14
- postgres: 14 - postgres: 14
os: macos-13 os: macos-13
steps: steps:
@@ -70,13 +70,12 @@ jobs:
tar xf $TAG.tar.gz tar xf $TAG.tar.gz
mv postgres-$TAG postgres mv postgres-$TAG postgres
env: env:
TAG: ${{ matrix.postgres == 17 && 'REL_17_2' || 'REL_14_15' }} TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl" - run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env: env:
PERL5LIB: /Users/runner/perl5/lib/perl5 PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make - run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
env: env:
LLVM_VERSION: ${{ matrix.os == 'macos-15' && 18 || 15 }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows: windows:
runs-on: windows-latest runs-on: windows-latest
@@ -126,7 +125,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1 - uses: ankane/setup-postgres-valgrind@v1
with: with:
postgres-version: 17 postgres-version: 16
check-ub: yes check-ub: yes
- run: make OPTFLAGS="" - run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install - run: sudo --preserve-env=PG_CONFIG make install

View File

@@ -1,4 +1,4 @@
## 0.8.0 (2024-10-30) ## 0.8.0 (unreleased)
- Added support for iterative index scans - Added support for iterative index scans
- Added casts for arrays to `sparsevec` - Added casts for arrays to `sparsevec`

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.0", "version": "0.7.4",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -12,7 +12,7 @@
"prereqs": { "prereqs": {
"runtime": { "runtime": {
"requires": { "requires": {
"PostgreSQL": "13.0.0" "PostgreSQL": "12.0.0"
} }
} }
}, },
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.8.0", "version": "0.7.4",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.0 EXTVERSION = 0.7.4
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql) DATA = $(wildcard sql/*--*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.0 EXTVERSION = 0.7.4
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

125
README.md
View File

@@ -17,11 +17,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
### Linux and Mac ### Linux and Mac
Compile and install the extension (supports Postgres 13+) Compile and install the extension (supports Postgres 12+)
```sh ```sh
cd /tmp cd /tmp
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -46,12 +46,14 @@ Then use `nmake` to build:
```cmd ```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16" set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP% cd %TEMP%
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
nmake /F Makefile.win nmake /F Makefile.win
nmake /F Makefile.win install nmake /F Makefile.win install
``` ```
Note: Postgres 17 is not supported yet due to an upstream issue
See the [installation notes](#installation-notes---windows) if you run into issues See the [installation notes](#installation-notes---windows) if you run into issues
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge). You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -82,7 +84,7 @@ Get the nearest neighbors by L2 distance
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
``` ```
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`) Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
@@ -146,9 +148,9 @@ Supported distance functions are:
- `<->` - L2 distance - `<->` - L2 distance
- `<#>` - (negative) inner product - `<#>` - (negative) inner product
- `<=>` - cosine distance - `<=>` - cosine distance
- `<+>` - L1 distance - `<+>` - L1 distance (added in 0.7.0)
- `<~>` - Hamming distance (binary vectors) - `<~>` - Hamming distance (binary vectors, added in 0.7.0)
- `<%>` - Jaccard distance (binary vectors) - `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
Get the nearest neighbors to a row Get the nearest neighbors to a row
@@ -235,19 +237,19 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops); CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
``` ```
L1 distance L1 distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops); CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
``` ```
Hamming distance Hamming distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops); CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
``` ```
Jaccard distance Jaccard distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops); CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
@@ -256,9 +258,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
Supported types are: Supported types are:
- `vector` - up to 2,000 dimensions - `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions - `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions - `bit` - up to 64,000 dimensions (added in 0.7.0)
- `sparsevec` - up to 1,000 non-zero elements - `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0)
### Index Options ### Index Options
@@ -312,7 +314,7 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t
Like other index types, its faster to create an index after loading your initial data Like other index types, its faster to create an index after loading your initial data
You can also speed up index creation by increasing the number of parallel workers (2 by default) Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql ```sql
SET max_parallel_maintenance_workers = 7; -- plus leader SET max_parallel_maintenance_workers = 7; -- plus leader
@@ -322,7 +324,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
### Indexing Progress ### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
```sql ```sql
SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index; SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -365,7 +367,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
``` ```
Hamming distance Hamming distance - added in 0.7.0
```sql ```sql
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
@@ -374,8 +376,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
Supported types are: Supported types are:
- `vector` - up to 2,000 dimensions - `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions - `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions - `bit` - up to 64,000 dimensions (added in 0.7.0)
### Query Options ### Query Options
@@ -408,7 +410,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
### Indexing Progress ### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
```sql ```sql
SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index; SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -475,7 +477,7 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
## Iterative Index Scans ## Iterative Index Scans
*Added in 0.8.0* *Unreleased*
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
@@ -547,6 +549,8 @@ Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used
## Half-Precision Vectors ## Half-Precision Vectors
*Added in 0.7.0*
Use the `halfvec` type to store half-precision vectors Use the `halfvec` type to store half-precision vectors
```sql ```sql
@@ -555,6 +559,8 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
## Half-Precision Indexing ## Half-Precision Indexing
*Added in 0.7.0*
Index vectors at half precision for smaller indexes Index vectors at half precision for smaller indexes
```sql ```sql
@@ -576,16 +582,24 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111'); INSERT INTO items (embedding) VALUES ('000'), ('111');
``` ```
Get the nearest neighbors by Hamming distance Get the nearest neighbors by Hamming distance (added in 0.7.0)
```sql ```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5; SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
``` ```
Or (before 0.7.0)
```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
```
Also supports Jaccard distance (`<%>`) Also supports Jaccard distance (`<%>`)
## Binary Quantization ## Binary Quantization
*Added in 0.7.0*
Use expression indexing for binary quantization Use expression indexing for binary quantization
```sql ```sql
@@ -608,6 +622,8 @@ SELECT * FROM (
## Sparse Vectors ## Sparse Vectors
*Added in 0.7.0*
Use the `sparsevec` type to store sparse vectors Use the `sparsevec` type to store sparse vectors
```sql ```sql
@@ -641,6 +657,8 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
## Indexing Subvectors ## Indexing Subvectors
*Added in 0.7.0*
Use expression indexing to index subvectors Use expression indexing to index subvectors
```sql ```sql
@@ -783,12 +801,8 @@ C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp) C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet) C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal) Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart) Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir) Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
Erlang | [pgvector-erlang](https://github.com/pgvector/pgvector-erlang)
Fortran | [pgvector-fortran](https://github.com/pgvector/pgvector-fortran)
Gleam | [pgvector-gleam](https://github.com/pgvector/pgvector-gleam)
Go | [pgvector-go](https://github.com/pgvector/pgvector-go) Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell) Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java) Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
@@ -802,7 +816,6 @@ Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php) PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python) Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r) R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor) Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust) Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift) Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
@@ -924,7 +937,7 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
#### Why are there less results for a query after adding an HNSW index? #### Why are there less results for a query after adding an HNSW index?
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`), which is 40 by default. There may be even less results due to dead tuples or filtering conditions in the query. Enabling [iterative index scans](#iterative-index-scans) can help address this. Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance). Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -936,7 +949,7 @@ The index was likely created with too little data for the number of lists. Drop
DROP INDEX index_name; DROP INDEX index_name;
``` ```
Results can also be limited by the number of probes (`ivfflat.probes`). Enabling [iterative index scans](#iterative-index-scans) can address this. Results can also be limited by the number of probes (`ivfflat.probes`).
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance). Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -1103,13 +1116,7 @@ Note: Replace `17` with your Postgres server version
### Missing SDK ### Missing SDK
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists. If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
```sh
pg_config --cppflags
```
Reinstall Postgres to fix this.
### Portability ### Portability
@@ -1127,14 +1134,6 @@ make OPTFLAGS=""
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct. If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
### Mismatched Architecture
If compilation fails with `error C2196: case value '4' already used`, make sure `vcvars64.bat` was called. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
### Missing Symbol
If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+.
### Permissions ### Permissions
If installation fails with `Access is denied`, re-run the installation instructions as an administrator. If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
@@ -1154,17 +1153,11 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually: You can also build the image manually:
```sh ```sh
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector . docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
``` ```
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
```
### Homebrew ### Homebrew
With Homebrew Postgres, you can use: With Homebrew Postgres, you can use:
@@ -1210,7 +1203,7 @@ Note: Replace `17` with your Postgres server version
Install the FreeBSD package with: Install the FreeBSD package with:
```sh ```sh
pkg install postgresql16-pgvector pkg install postgresql15-pgvector
``` ```
or the port with: or the port with:
@@ -1252,6 +1245,36 @@ You can check the version in the current database with:
SELECT extversion FROM pg_extension WHERE extname = 'vector'; SELECT extversion FROM pg_extension WHERE extname = 'vector';
``` ```
## Upgrade Notes
### 0.6.0
#### Postgres 12
If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
```sql
ALTER TYPE vector SET (STORAGE = external);
```
Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
#### Docker
The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
```sh
docker pull pgvector/pgvector:pg16
# or
docker pull pgvector/pgvector:0.6.0-pg16
```
Also, if youve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
```
## Thanks ## Thanks
Thanks to: Thanks to:

View File

@@ -8,9 +8,6 @@
#error "Requires PostgreSQL 13+" #error "Requires PostgreSQL 13+"
#endif #endif
/* Check architecture in first header */
StaticAssertDecl(sizeof(Datum) == SIZEOF_DATUM, "Architecture mismatch");
extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance); extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);
extern double (*BitJaccardDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb); extern double (*BitJaccardDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb);

View File

@@ -126,7 +126,7 @@ typedef struct HnswNeighborArray HnswNeighborArray;
#define HnswPtrDeclare(type, relptrtype, ptrtype) \ #define HnswPtrDeclare(type, relptrtype, ptrtype) \
relptr_declare(type, relptrtype); \ relptr_declare(type, relptrtype); \
typedef union { type *ptr; relptrtype relptr; } ptrtype typedef union { type *ptr; relptrtype relptr; } ptrtype;
/* Pointers that can be absolute or relative */ /* Pointers that can be absolute or relative */
/* Use char for DatumPtr so works with Pointer */ /* Use char for DatumPtr so works with Pointer */

View File

@@ -722,6 +722,8 @@ CountElement(HnswElement skipElement, HnswElement e)
static void static void
HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize) HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize)
{ {
uint32 hashes[HNSW_MAX_M * 2];
/* Get the neighborhood at layer lc */ /* Get the neighborhood at layer lc */
HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc); HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc);
@@ -733,14 +735,33 @@ HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unv
*unvisitedLength = 0; *unvisitedLength = 0;
for (int i = 0; i < localNeighborhood->length; i++) for (int i = 0; i < localNeighborhood->length; i++)
hashes[i] = HnswPtrAccess(base, localNeighborhood->items[i].element)->hash;
if (base != NULL)
{ {
HnswCandidate *hc = &localNeighborhood->items[i]; for (int i = 0; i < localNeighborhood->length; i++)
bool found; {
HnswCandidate *hc = &localNeighborhood->items[i];
bool found;
AddToVisited(base, v, hc->element, true, &found); offsethash_insert_hash(v->offsets, HnswPtrOffset(hc->element), hashes[i], &found);
if (!found) if (!found)
unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element); unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
}
}
else
{
for (int i = 0; i < localNeighborhood->length; i++)
{
HnswCandidate *hc = &localNeighborhood->items[i];
bool found;
pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), hashes[i], &found);
if (!found)
unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
}
} }
} }
@@ -1393,7 +1414,7 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support); FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
Datum Datum
@@ -1406,7 +1427,7 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support); FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
Datum Datum
@@ -1419,4 +1440,4 @@ hnsw_sparsevec_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };

View File

@@ -9,10 +9,6 @@
#include "storage/lmgr.h" #include "storage/lmgr.h"
#include "utils/memutils.h" #include "utils/memutils.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/* /*
* Check if deleted list contains an index TID * Check if deleted list contains an index TID
*/ */

View File

@@ -360,7 +360,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->sortdesc = CreateTemplateTupleDesc(3); buildstate->sortdesc = CreateTemplateTupleDesc(3);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", buildstate->tupdesc->attrs[0].atttypid, -1, 0);
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual); buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);

View File

@@ -355,7 +355,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support); FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
Datum Datum
@@ -370,4 +370,4 @@ ivfflat_bit_support(PG_FUNCTION_ARGS)
}; };
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
} };

View File

@@ -5,10 +5,6 @@
#include "ivfflat.h" #include "ivfflat.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/* /*
* Bulk delete tuples from the index * Bulk delete tuples from the index
*/ */

View File

@@ -4,7 +4,6 @@
#include <math.h> #include <math.h>
#include "catalog/pg_type.h" #include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "common/string.h" #include "common/string.h"
#include "fmgr.h" #include "fmgr.h"
#include "halfutils.h" #include "halfutils.h"
@@ -13,10 +12,17 @@
#include "sparsevec.h" #include "sparsevec.h"
#include "utils/array.h" #include "utils/array.h"
#include "utils/builtins.h" #include "utils/builtins.h"
#include "utils/float.h"
#include "utils/lsyscache.h" #include "utils/lsyscache.h"
#include "vector.h" #include "vector.h"
#if PG_VERSION_NUM >= 120000
#include "common/shortest_dec.h"
#include "utils/float.h"
#else
#include <float.h>
#include "utils/builtins.h"
#endif
typedef struct SparseInputElement typedef struct SparseInputElement
{ {
int32 index; int32 index;

View File

@@ -123,12 +123,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[0,0,0] [0,0,0]
(3 rows) (3 rows)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET hnsw.iterative_scan; RESET hnsw.iterative_scan;
RESET hnsw.ef_search; RESET hnsw.ef_search;
DROP TABLE t; DROP TABLE t;

View File

@@ -110,15 +110,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[1,1,1] [1,1,1]
(2 rows) (2 rows)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET ivfflat.iterative_scan; RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes; RESET ivfflat.max_probes;
DROP TABLE t; DROP TABLE t;

View File

@@ -70,9 +70,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SET hnsw.iterative_scan = relaxed_order; SET hnsw.iterative_scan = relaxed_order;
SELECT * FROM t ORDER BY val <-> '[3,3,3]'; SELECT * FROM t ORDER BY val <-> '[3,3,3]';
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
RESET hnsw.iterative_scan; RESET hnsw.iterative_scan;
RESET hnsw.ef_search; RESET hnsw.ef_search;
DROP TABLE t; DROP TABLE t;

View File

@@ -59,9 +59,6 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SET ivfflat.max_probes = 2; SET ivfflat.max_probes = 2;
SELECT * FROM t ORDER BY val <-> '[3,3,3]'; SELECT * FROM t ORDER BY val <-> '[3,3,3]';
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
RESET ivfflat.iterative_scan; RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes; RESET ivfflat.max_probes;
DROP TABLE t; DROP TABLE t;

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods' comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.0' default_version = '0.7.4'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true