mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-23 04:20:56 +08:00
Compare commits
1 Commits
subvector
...
hnsw-debug
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fb2782c9f6 |
1
.github/workflows/build.yml
vendored
1
.github/workflows/build.yml
vendored
@@ -73,7 +73,6 @@ jobs:
|
|||||||
postgres-version: 14
|
postgres-version: 14
|
||||||
- run: |
|
- run: |
|
||||||
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
|
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
|
||||||
cd %TEMP% && ^
|
|
||||||
nmake /NOLOGO /F Makefile.win && ^
|
nmake /NOLOGO /F Makefile.win && ^
|
||||||
nmake /NOLOGO /F Makefile.win install && ^
|
nmake /NOLOGO /F Makefile.win install && ^
|
||||||
nmake /NOLOGO /F Makefile.win installcheck && ^
|
nmake /NOLOGO /F Makefile.win installcheck && ^
|
||||||
|
|||||||
10
CHANGELOG.md
10
CHANGELOG.md
@@ -1,15 +1,7 @@
|
|||||||
## 0.7.0 (unreleased)
|
|
||||||
|
|
||||||
- Added `subvector` function
|
|
||||||
|
|
||||||
## 0.6.2 (2024-03-18)
|
|
||||||
|
|
||||||
- Reduced lock contention with parallel HNSW index builds
|
|
||||||
|
|
||||||
## 0.6.1 (2024-03-04)
|
## 0.6.1 (2024-03-04)
|
||||||
|
|
||||||
- Fixed error with `ANALYZE` and vectors with different dimensions
|
- Fixed error with `ANALYZE` and vectors with different dimensions
|
||||||
- Fixed segmentation fault with `shared_preload_libraries`
|
- Fixed error with `shared_preload_libraries`
|
||||||
- Fixed vector subtraction being marked as commutative
|
- Fixed vector subtraction being marked as commutative
|
||||||
|
|
||||||
## 0.6.0 (2024-01-29)
|
## 0.6.0 (2024-01-29)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
"name": "vector",
|
"name": "vector",
|
||||||
"abstract": "Open-source vector similarity search for Postgres",
|
"abstract": "Open-source vector similarity search for Postgres",
|
||||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||||
"version": "0.6.2",
|
"version": "0.6.1",
|
||||||
"maintainer": [
|
"maintainer": [
|
||||||
"Andrew Kane <andrew@ankane.org>"
|
"Andrew Kane <andrew@ankane.org>"
|
||||||
],
|
],
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
"vector": {
|
"vector": {
|
||||||
"file": "sql/vector.sql",
|
"file": "sql/vector.sql",
|
||||||
"docfile": "README.md",
|
"docfile": "README.md",
|
||||||
"version": "0.6.2",
|
"version": "0.6.1",
|
||||||
"abstract": "Open-source vector similarity search for Postgres"
|
"abstract": "Open-source vector similarity search for Postgres"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
2
Makefile
2
Makefile
@@ -1,5 +1,5 @@
|
|||||||
EXTENSION = vector
|
EXTENSION = vector
|
||||||
EXTVERSION = 0.6.2
|
EXTVERSION = 0.6.1
|
||||||
|
|
||||||
MODULE_big = vector
|
MODULE_big = vector
|
||||||
DATA = $(wildcard sql/*--*.sql)
|
DATA = $(wildcard sql/*--*.sql)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
EXTENSION = vector
|
EXTENSION = vector
|
||||||
EXTVERSION = 0.6.2
|
EXTVERSION = 0.6.1
|
||||||
|
|
||||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||||
HEADERS = src\vector.h
|
HEADERS = src\vector.h
|
||||||
|
|||||||
129
README.md
129
README.md
@@ -20,13 +20,13 @@ Compile and install the extension (supports Postgres 12+)
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
cd /tmp
|
cd /tmp
|
||||||
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
|
git clone --branch v0.6.1 https://github.com/pgvector/pgvector.git
|
||||||
cd pgvector
|
cd pgvector
|
||||||
make
|
make
|
||||||
make install # may need sudo
|
make install # may need sudo
|
||||||
```
|
```
|
||||||
|
|
||||||
See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
|
See the [installation notes](#installation-notes) if you run into issues
|
||||||
|
|
||||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
|
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
|
||||||
|
|
||||||
@@ -44,15 +44,12 @@ Then use `nmake` to build:
|
|||||||
|
|
||||||
```cmd
|
```cmd
|
||||||
set "PGROOT=C:\Program Files\PostgreSQL\16"
|
set "PGROOT=C:\Program Files\PostgreSQL\16"
|
||||||
cd %TEMP%
|
git clone --branch v0.6.1 https://github.com/pgvector/pgvector.git
|
||||||
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
|
|
||||||
cd pgvector
|
cd pgvector
|
||||||
nmake /F Makefile.win
|
nmake /F Makefile.win
|
||||||
nmake /F Makefile.win install
|
nmake /F Makefile.win install
|
||||||
```
|
```
|
||||||
|
|
||||||
See the [installation notes](#installation-notes---windows) if you run into issues
|
|
||||||
|
|
||||||
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
|
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
@@ -413,39 +410,13 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
|
|||||||
|
|
||||||
## Performance
|
## Performance
|
||||||
|
|
||||||
### Tuning
|
|
||||||
|
|
||||||
Use a tool like [PgTune](https://pgtune.leopard.in.ua/) to set initial values for Postgres server parameters.
|
|
||||||
|
|
||||||
### Loading
|
|
||||||
|
|
||||||
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py)).
|
|
||||||
|
|
||||||
```sql
|
|
||||||
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
|
|
||||||
```
|
|
||||||
|
|
||||||
Add any indexes *after* loading the initial data for best performance.
|
|
||||||
|
|
||||||
### Indexing
|
|
||||||
|
|
||||||
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
|
|
||||||
|
|
||||||
In production environments, create indexes concurrently to avoid blocking writes.
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE INDEX CONCURRENTLY ...
|
|
||||||
```
|
|
||||||
|
|
||||||
### Querying
|
|
||||||
|
|
||||||
Use `EXPLAIN ANALYZE` to debug performance.
|
Use `EXPLAIN ANALYZE` to debug performance.
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Exact Search
|
### Exact Search
|
||||||
|
|
||||||
To speed up queries without an index, increase `max_parallel_workers_per_gather`.
|
To speed up queries without an index, increase `max_parallel_workers_per_gather`.
|
||||||
|
|
||||||
@@ -459,7 +430,7 @@ If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform
|
|||||||
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Approximate Search
|
### Approximate Search
|
||||||
|
|
||||||
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
|
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
|
||||||
|
|
||||||
@@ -467,7 +438,7 @@ To speed up queries with an IVFFlat index, increase the number of inverted lists
|
|||||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||||
```
|
```
|
||||||
|
|
||||||
### Vacuuming
|
## Vacuuming
|
||||||
|
|
||||||
Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
|
Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
|
||||||
|
|
||||||
@@ -476,41 +447,6 @@ REINDEX INDEX CONCURRENTLY index_name;
|
|||||||
VACUUM table_name;
|
VACUUM table_name;
|
||||||
```
|
```
|
||||||
|
|
||||||
## Monitoring
|
|
||||||
|
|
||||||
Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE EXTENSION pg_stat_statements;
|
|
||||||
```
|
|
||||||
|
|
||||||
Get the most time-consuming queries with:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
|
|
||||||
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
|
|
||||||
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
|
|
||||||
```
|
|
||||||
|
|
||||||
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
|
|
||||||
|
|
||||||
Monitor recall by comparing results from approximate search with exact search.
|
|
||||||
|
|
||||||
```sql
|
|
||||||
BEGIN;
|
|
||||||
SET LOCAL enable_indexscan = off; -- use exact search
|
|
||||||
SELECT ...
|
|
||||||
COMMIT;
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scaling
|
|
||||||
|
|
||||||
Scale pgvector the same way you scale Postgres.
|
|
||||||
|
|
||||||
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
|
|
||||||
|
|
||||||
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
|
|
||||||
|
|
||||||
## Languages
|
## Languages
|
||||||
|
|
||||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||||
@@ -604,18 +540,6 @@ and query with:
|
|||||||
SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5;
|
SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5;
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Are binary vectors supported?
|
|
||||||
|
|
||||||
You can store binary vectors and perform exact nearest neighbor search by Hamming distance in Postgres without an extension ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py)).
|
|
||||||
|
|
||||||
```tsql
|
|
||||||
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
|
|
||||||
INSERT INTO items (embedding) VALUES (B'000'), (B'111');
|
|
||||||
SELECT * FROM items ORDER BY bit_count(embedding # B'101') LIMIT 5;
|
|
||||||
```
|
|
||||||
|
|
||||||
Indexing is not currently supported.
|
|
||||||
|
|
||||||
#### Do indexes need to fit into memory?
|
#### Do indexes need to fit into memory?
|
||||||
|
|
||||||
No, but like other index types, you’ll likely see better performance if they do. You can get the size of an index with:
|
No, but like other index types, you’ll likely see better performance if they do. You can get the size of an index with:
|
||||||
@@ -628,17 +552,7 @@ SELECT pg_size_pretty(pg_relation_size('index_name'));
|
|||||||
|
|
||||||
#### Why isn’t a query using an index?
|
#### Why isn’t a query using an index?
|
||||||
|
|
||||||
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator, not an expression.
|
The cost estimation in pgvector < 0.4.3 does not always work well with the planner. You can encourage the planner to use an index for a query with:
|
||||||
|
|
||||||
```sql
|
|
||||||
-- index
|
|
||||||
ORDER BY embedding <=> '[3,1,2]' LIMIT 5;
|
|
||||||
|
|
||||||
-- no index
|
|
||||||
ORDER BY 1 - (embedding <=> '[3,1,2]') DESC LIMIT 5;
|
|
||||||
```
|
|
||||||
|
|
||||||
You can encourage the planner to use an index for a query with:
|
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
BEGIN;
|
BEGIN;
|
||||||
@@ -706,7 +620,6 @@ cosine_distance(vector, vector) → double precision | cosine distance |
|
|||||||
inner_product(vector, vector) → double precision | inner product |
|
inner_product(vector, vector) → double precision | inner product |
|
||||||
l2_distance(vector, vector) → double precision | Euclidean distance |
|
l2_distance(vector, vector) → double precision | Euclidean distance |
|
||||||
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
|
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
|
||||||
subvector(vector, integer, integer) → vector | subvector | 0.7.0 [unreleased]
|
|
||||||
vector_dims(vector) → integer | number of dimensions |
|
vector_dims(vector) → integer | number of dimensions |
|
||||||
vector_norm(vector) → double precision | Euclidean norm |
|
vector_norm(vector) → double precision | Euclidean norm |
|
||||||
|
|
||||||
@@ -717,7 +630,7 @@ Function | Description | Added
|
|||||||
avg(vector) → vector | average |
|
avg(vector) → vector | average |
|
||||||
sum(vector) → vector | sum | 0.5.0
|
sum(vector) → vector | sum | 0.5.0
|
||||||
|
|
||||||
## Installation Notes - Linux and Mac
|
## Installation Notes
|
||||||
|
|
||||||
### Postgres Location
|
### Postgres Location
|
||||||
|
|
||||||
@@ -759,24 +672,12 @@ If compilation fails and the output includes `warning: no such sysroot directory
|
|||||||
|
|
||||||
### Portability
|
### Portability
|
||||||
|
|
||||||
By default, pgvector compiles with `-march=native` on some platforms for best performance. However, this can lead to `Illegal instruction` errors if trying to run the compiled extension on a different machine.
|
|
||||||
|
|
||||||
To compile for portability, use:
|
To compile for portability, use:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
make OPTFLAGS=""
|
make OPTFLAGS=""
|
||||||
```
|
```
|
||||||
|
|
||||||
## Installation Notes - Windows
|
|
||||||
|
|
||||||
### Missing Header
|
|
||||||
|
|
||||||
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
|
|
||||||
|
|
||||||
### Permissions
|
|
||||||
|
|
||||||
If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
|
|
||||||
|
|
||||||
## Additional Installation Methods
|
## Additional Installation Methods
|
||||||
|
|
||||||
### Docker
|
### Docker
|
||||||
@@ -792,7 +693,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
|
|||||||
You can also build the image manually:
|
You can also build the image manually:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
|
git clone --branch v0.6.1 https://github.com/pgvector/pgvector.git
|
||||||
cd pgvector
|
cd pgvector
|
||||||
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
|
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
|
||||||
```
|
```
|
||||||
@@ -961,12 +862,6 @@ make installcheck REGRESS=functions # regression test
|
|||||||
make prove_installcheck PROVE_TESTS=test/t/001_ivfflat_wal.pl # TAP test
|
make prove_installcheck PROVE_TESTS=test/t/001_ivfflat_wal.pl # TAP test
|
||||||
```
|
```
|
||||||
|
|
||||||
To enable assertions:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
|
|
||||||
```
|
|
||||||
|
|
||||||
To enable benchmarking:
|
To enable benchmarking:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -979,6 +874,12 @@ To show memory usage:
|
|||||||
make clean && PG_CFLAGS="-DHNSW_MEMORY -DIVFFLAT_MEMORY" make && make install
|
make clean && PG_CFLAGS="-DHNSW_MEMORY -DIVFFLAT_MEMORY" make && make install
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To enable assertions:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
|
||||||
|
```
|
||||||
|
|
||||||
To get k-means metrics:
|
To get k-means metrics:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
|
||||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.6.2'" to load this file. \quit
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
|
||||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.0'" to load this file. \quit
|
|
||||||
|
|
||||||
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
|
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
|
||||||
@@ -58,9 +58,6 @@ CREATE FUNCTION vector_sub(vector, vector) RETURNS vector
|
|||||||
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
|
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
|
|
||||||
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
|
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
|
||||||
|
|
||||||
-- private functions
|
-- private functions
|
||||||
|
|
||||||
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
|
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
|
||||||
|
|||||||
24
src/hnsw.c
24
src/hnsw.c
@@ -17,7 +17,10 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
int hnsw_ef_search;
|
int hnsw_ef_search;
|
||||||
int hnsw_lock_tranche_id;
|
int hnsw_entry_lock_tranche_id;
|
||||||
|
int hnsw_allocator_lock_tranche_id;
|
||||||
|
int hnsw_flush_lock_tranche_id;
|
||||||
|
int hnsw_element_lock_tranche_id;
|
||||||
static relopt_kind hnsw_relopt_kind;
|
static relopt_kind hnsw_relopt_kind;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -36,16 +39,27 @@ HnswInitLockTranche(void)
|
|||||||
bool found;
|
bool found;
|
||||||
|
|
||||||
LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
|
LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
|
||||||
tranche_ids = ShmemInitStruct("hnsw LWLock ids",
|
tranche_ids = ShmemInitStruct("hnsw LWLock ids v2",
|
||||||
sizeof(int) * 1,
|
sizeof(int) * 4,
|
||||||
&found);
|
&found);
|
||||||
if (!found)
|
if (!found)
|
||||||
|
{
|
||||||
tranche_ids[0] = LWLockNewTrancheId();
|
tranche_ids[0] = LWLockNewTrancheId();
|
||||||
hnsw_lock_tranche_id = tranche_ids[0];
|
tranche_ids[1] = LWLockNewTrancheId();
|
||||||
|
tranche_ids[2] = LWLockNewTrancheId();
|
||||||
|
tranche_ids[3] = LWLockNewTrancheId();
|
||||||
|
}
|
||||||
|
hnsw_entry_lock_tranche_id = tranche_ids[0];
|
||||||
|
hnsw_allocator_lock_tranche_id = tranche_ids[1];
|
||||||
|
hnsw_flush_lock_tranche_id = tranche_ids[2];
|
||||||
|
hnsw_element_lock_tranche_id = tranche_ids[3];
|
||||||
LWLockRelease(AddinShmemInitLock);
|
LWLockRelease(AddinShmemInitLock);
|
||||||
|
|
||||||
/* Per-backend registration of the tranche ID */
|
/* Per-backend registration of the tranche ID */
|
||||||
LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
|
LWLockRegisterTranche(hnsw_entry_lock_tranche_id, "HnswEntry");
|
||||||
|
LWLockRegisterTranche(hnsw_allocator_lock_tranche_id, "HnswAllocator");
|
||||||
|
LWLockRegisterTranche(hnsw_flush_lock_tranche_id, "HnswFlush");
|
||||||
|
LWLockRegisterTranche(hnsw_element_lock_tranche_id, "HnswElement");
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
14
src/hnsw.h
14
src/hnsw.h
@@ -113,7 +113,10 @@
|
|||||||
|
|
||||||
/* Variables */
|
/* Variables */
|
||||||
extern int hnsw_ef_search;
|
extern int hnsw_ef_search;
|
||||||
extern int hnsw_lock_tranche_id;
|
extern int hnsw_entry_lock_tranche_id;
|
||||||
|
extern int hnsw_allocator_lock_tranche_id;
|
||||||
|
extern int hnsw_flush_lock_tranche_id;
|
||||||
|
extern int hnsw_element_lock_tranche_id;
|
||||||
|
|
||||||
typedef struct HnswElementData HnswElementData;
|
typedef struct HnswElementData HnswElementData;
|
||||||
typedef struct HnswNeighborArray HnswNeighborArray;
|
typedef struct HnswNeighborArray HnswNeighborArray;
|
||||||
@@ -129,7 +132,7 @@ HnswPtrDeclare(HnswNeighborArray, HnswNeighborArrayRelptr, HnswNeighborArrayPtr)
|
|||||||
HnswPtrDeclare(HnswNeighborArrayPtr, HnswNeighborsRelptr, HnswNeighborsPtr);
|
HnswPtrDeclare(HnswNeighborArrayPtr, HnswNeighborsRelptr, HnswNeighborsPtr);
|
||||||
HnswPtrDeclare(char, DatumRelptr, DatumPtr);
|
HnswPtrDeclare(char, DatumRelptr, DatumPtr);
|
||||||
|
|
||||||
struct HnswElementData
|
typedef struct HnswElementData
|
||||||
{
|
{
|
||||||
HnswElementPtr next;
|
HnswElementPtr next;
|
||||||
ItemPointerData heaptids[HNSW_HEAPTIDS];
|
ItemPointerData heaptids[HNSW_HEAPTIDS];
|
||||||
@@ -144,7 +147,7 @@ struct HnswElementData
|
|||||||
BlockNumber neighborPage;
|
BlockNumber neighborPage;
|
||||||
DatumPtr value;
|
DatumPtr value;
|
||||||
LWLock lock;
|
LWLock lock;
|
||||||
};
|
} HnswElementData;
|
||||||
|
|
||||||
typedef HnswElementData * HnswElement;
|
typedef HnswElementData * HnswElement;
|
||||||
|
|
||||||
@@ -155,12 +158,12 @@ typedef struct HnswCandidate
|
|||||||
bool closer;
|
bool closer;
|
||||||
} HnswCandidate;
|
} HnswCandidate;
|
||||||
|
|
||||||
struct HnswNeighborArray
|
typedef struct HnswNeighborArray
|
||||||
{
|
{
|
||||||
int length;
|
int length;
|
||||||
bool closerSet;
|
bool closerSet;
|
||||||
HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
|
HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
|
||||||
};
|
} HnswNeighborArray;
|
||||||
|
|
||||||
typedef struct HnswPairingHeapNode
|
typedef struct HnswPairingHeapNode
|
||||||
{
|
{
|
||||||
@@ -185,7 +188,6 @@ typedef struct HnswGraph
|
|||||||
|
|
||||||
/* Entry state */
|
/* Entry state */
|
||||||
LWLock entryLock;
|
LWLock entryLock;
|
||||||
LWLock entryWaitLock;
|
|
||||||
HnswElementPtr entryPoint;
|
HnswElementPtr entryPoint;
|
||||||
|
|
||||||
/* Allocations state */
|
/* Allocations state */
|
||||||
|
|||||||
@@ -431,15 +431,10 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
|
|||||||
HnswGraph *graph = buildstate->graph;
|
HnswGraph *graph = buildstate->graph;
|
||||||
HnswElement entryPoint;
|
HnswElement entryPoint;
|
||||||
LWLock *entryLock = &graph->entryLock;
|
LWLock *entryLock = &graph->entryLock;
|
||||||
LWLock *entryWaitLock = &graph->entryWaitLock;
|
|
||||||
int efConstruction = buildstate->efConstruction;
|
int efConstruction = buildstate->efConstruction;
|
||||||
int m = buildstate->m;
|
int m = buildstate->m;
|
||||||
char *base = buildstate->hnswarea;
|
char *base = buildstate->hnswarea;
|
||||||
|
|
||||||
/* Wait if another process needs exclusive lock on entry lock */
|
|
||||||
LWLockAcquire(entryWaitLock, LW_EXCLUSIVE);
|
|
||||||
LWLockRelease(entryWaitLock);
|
|
||||||
|
|
||||||
/* Get entry point */
|
/* Get entry point */
|
||||||
LWLockAcquire(entryLock, LW_SHARED);
|
LWLockAcquire(entryLock, LW_SHARED);
|
||||||
entryPoint = HnswPtrAccess(base, graph->entryPoint);
|
entryPoint = HnswPtrAccess(base, graph->entryPoint);
|
||||||
@@ -450,10 +445,8 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
|
|||||||
/* Release shared lock */
|
/* Release shared lock */
|
||||||
LWLockRelease(entryLock);
|
LWLockRelease(entryLock);
|
||||||
|
|
||||||
/* Tell other processes to wait and get exclusive lock */
|
/* Get exclusive lock */
|
||||||
LWLockAcquire(entryWaitLock, LW_EXCLUSIVE);
|
|
||||||
LWLockAcquire(entryLock, LW_EXCLUSIVE);
|
LWLockAcquire(entryLock, LW_EXCLUSIVE);
|
||||||
LWLockRelease(entryWaitLock);
|
|
||||||
|
|
||||||
/* Get latest entry point after lock is acquired */
|
/* Get latest entry point after lock is acquired */
|
||||||
entryPoint = HnswPtrAccess(base, graph->entryPoint);
|
entryPoint = HnswPtrAccess(base, graph->entryPoint);
|
||||||
@@ -555,7 +548,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
|||||||
HnswPtrStore(base, element->value, valuePtr);
|
HnswPtrStore(base, element->value, valuePtr);
|
||||||
|
|
||||||
/* Create a lock for the element */
|
/* Create a lock for the element */
|
||||||
LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
|
LWLockInitialize(&element->lock, hnsw_element_lock_tranche_id);
|
||||||
|
|
||||||
/* Insert tuple */
|
/* Insert tuple */
|
||||||
InsertTupleInMemory(buildstate, element);
|
InsertTupleInMemory(buildstate, element);
|
||||||
@@ -618,10 +611,9 @@ InitGraph(HnswGraph * graph, char *base, long memoryTotal)
|
|||||||
graph->flushed = false;
|
graph->flushed = false;
|
||||||
graph->indtuples = 0;
|
graph->indtuples = 0;
|
||||||
SpinLockInit(&graph->lock);
|
SpinLockInit(&graph->lock);
|
||||||
LWLockInitialize(&graph->entryLock, hnsw_lock_tranche_id);
|
LWLockInitialize(&graph->entryLock, hnsw_entry_lock_tranche_id);
|
||||||
LWLockInitialize(&graph->entryWaitLock, hnsw_lock_tranche_id);
|
LWLockInitialize(&graph->allocatorLock, hnsw_allocator_lock_tranche_id);
|
||||||
LWLockInitialize(&graph->allocatorLock, hnsw_lock_tranche_id);
|
LWLockInitialize(&graph->flushLock, hnsw_flush_lock_tranche_id);
|
||||||
LWLockInitialize(&graph->flushLock, hnsw_lock_tranche_id);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
31
src/vector.c
31
src/vector.c
@@ -860,37 +860,6 @@ vector_mul(PG_FUNCTION_ARGS)
|
|||||||
PG_RETURN_POINTER(result);
|
PG_RETURN_POINTER(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Get a subset of a vector
|
|
||||||
*/
|
|
||||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
|
|
||||||
Datum
|
|
||||||
subvector(PG_FUNCTION_ARGS)
|
|
||||||
{
|
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
|
||||||
int32 start = PG_GETARG_INT32(1);
|
|
||||||
int32 count = PG_GETARG_INT32(2);
|
|
||||||
int32 end = start + count;
|
|
||||||
float *ax = a->x;
|
|
||||||
Vector *result;
|
|
||||||
int dim;
|
|
||||||
|
|
||||||
if (start < 1)
|
|
||||||
start = 1;
|
|
||||||
|
|
||||||
if (end > a->dim)
|
|
||||||
end = a->dim + 1;
|
|
||||||
|
|
||||||
dim = end - start;
|
|
||||||
CheckDim(dim);
|
|
||||||
result = InitVector(dim);
|
|
||||||
|
|
||||||
for (int i = 0; i < dim; i++)
|
|
||||||
result->x[i] = ax[start - 1 + i];
|
|
||||||
|
|
||||||
PG_RETURN_POINTER(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Internal helper to compare vectors
|
* Internal helper to compare vectors
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -208,34 +208,6 @@ SELECT l1_distance('[3e38]', '[-3e38]');
|
|||||||
Infinity
|
Infinity
|
||||||
(1 row)
|
(1 row)
|
||||||
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 1, 3);
|
|
||||||
subvector
|
|
||||||
-----------
|
|
||||||
[1,2,3]
|
|
||||||
(1 row)
|
|
||||||
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 3, 2);
|
|
||||||
subvector
|
|
||||||
-----------
|
|
||||||
[3,4]
|
|
||||||
(1 row)
|
|
||||||
|
|
||||||
SELECT subvector('[1,2,3,4,5]', -1, 3);
|
|
||||||
subvector
|
|
||||||
-----------
|
|
||||||
[1]
|
|
||||||
(1 row)
|
|
||||||
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 3, 9);
|
|
||||||
subvector
|
|
||||||
-----------
|
|
||||||
[3,4,5]
|
|
||||||
(1 row)
|
|
||||||
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 1, 0);
|
|
||||||
ERROR: vector must have at least 1 dimension
|
|
||||||
SELECT subvector('[1,2,3,4,5]', -1, 2);
|
|
||||||
ERROR: vector must have at least 1 dimension
|
|
||||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||||
avg
|
avg
|
||||||
-----------
|
-----------
|
||||||
|
|||||||
@@ -116,30 +116,8 @@ SELECT '[1, ,3]'::vector;
|
|||||||
ERROR: invalid input syntax for type vector: "[1, ,3]"
|
ERROR: invalid input syntax for type vector: "[1, ,3]"
|
||||||
LINE 1: SELECT '[1, ,3]'::vector;
|
LINE 1: SELECT '[1, ,3]'::vector;
|
||||||
^
|
^
|
||||||
SELECT '[1,2,3]'::vector(3);
|
|
||||||
vector
|
|
||||||
---------
|
|
||||||
[1,2,3]
|
|
||||||
(1 row)
|
|
||||||
|
|
||||||
SELECT '[1,2,3]'::vector(2);
|
SELECT '[1,2,3]'::vector(2);
|
||||||
ERROR: expected 2 dimensions, not 3
|
ERROR: expected 2 dimensions, not 3
|
||||||
SELECT '[1,2,3]'::vector(3, 2);
|
|
||||||
ERROR: invalid type modifier
|
|
||||||
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
|
|
||||||
^
|
|
||||||
SELECT '[1,2,3]'::vector('a');
|
|
||||||
ERROR: invalid input syntax for type integer: "a"
|
|
||||||
LINE 1: SELECT '[1,2,3]'::vector('a');
|
|
||||||
^
|
|
||||||
SELECT '[1,2,3]'::vector(0);
|
|
||||||
ERROR: dimensions for type vector must be at least 1
|
|
||||||
LINE 1: SELECT '[1,2,3]'::vector(0);
|
|
||||||
^
|
|
||||||
SELECT '[1,2,3]'::vector(16001);
|
|
||||||
ERROR: dimensions for type vector cannot exceed 16000
|
|
||||||
LINE 1: SELECT '[1,2,3]'::vector(16001);
|
|
||||||
^
|
|
||||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
||||||
unnest
|
unnest
|
||||||
---------
|
---------
|
||||||
|
|||||||
@@ -48,13 +48,6 @@ SELECT l1_distance('[0,0]', '[0,1]');
|
|||||||
SELECT l1_distance('[1,2]', '[3]');
|
SELECT l1_distance('[1,2]', '[3]');
|
||||||
SELECT l1_distance('[3e38]', '[-3e38]');
|
SELECT l1_distance('[3e38]', '[-3e38]');
|
||||||
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 1, 3);
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 3, 2);
|
|
||||||
SELECT subvector('[1,2,3,4,5]', -1, 3);
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 3, 9);
|
|
||||||
SELECT subvector('[1,2,3,4,5]', 1, 0);
|
|
||||||
SELECT subvector('[1,2,3,4,5]', -1, 2);
|
|
||||||
|
|
||||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
||||||
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||||
|
|||||||
@@ -22,13 +22,7 @@ SELECT '[1,]'::vector;
|
|||||||
SELECT '[1a]'::vector;
|
SELECT '[1a]'::vector;
|
||||||
SELECT '[1,,3]'::vector;
|
SELECT '[1,,3]'::vector;
|
||||||
SELECT '[1, ,3]'::vector;
|
SELECT '[1, ,3]'::vector;
|
||||||
|
|
||||||
SELECT '[1,2,3]'::vector(3);
|
|
||||||
SELECT '[1,2,3]'::vector(2);
|
SELECT '[1,2,3]'::vector(2);
|
||||||
SELECT '[1,2,3]'::vector(3, 2);
|
|
||||||
SELECT '[1,2,3]'::vector('a');
|
|
||||||
SELECT '[1,2,3]'::vector(0);
|
|
||||||
SELECT '[1,2,3]'::vector(16001);
|
|
||||||
|
|
||||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
||||||
SELECT '{"[1,2,3]"}'::vector(2)[];
|
SELECT '{"[1,2,3]"}'::vector(2)[];
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
comment = 'vector data type and ivfflat and hnsw access methods'
|
comment = 'vector data type and ivfflat and hnsw access methods'
|
||||||
default_version = '0.6.2'
|
default_version = '0.6.1'
|
||||||
module_pathname = '$libdir/vector'
|
module_pathname = '$libdir/vector'
|
||||||
relocatable = true
|
relocatable = true
|
||||||
|
|||||||
Reference in New Issue
Block a user