mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
1 Commits
angular_di
...
hnsw-print
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d866844d3 |
10
CHANGELOG.md
10
CHANGELOG.md
@@ -1,17 +1,13 @@
|
||||
## 0.5.1 (unreleased)
|
||||
|
||||
- Added `angular_distance` function
|
||||
|
||||
## 0.5.0 (2023-08-28)
|
||||
## 0.5.0 (unreleased)
|
||||
|
||||
- Added HNSW index type
|
||||
- Added support for parallel index builds for IVFFlat
|
||||
- Added support for parallel index builds
|
||||
- Added `l1_distance` function
|
||||
- Added element-wise multiplication for vectors
|
||||
- Added `sum` aggregate
|
||||
- Improved performance of distance functions
|
||||
- Fixed out of range results for cosine distance
|
||||
- Fixed results for NULL and NaN distances for IVFFlat
|
||||
- Fixed results for NULL and NaN distances
|
||||
|
||||
## 0.4.4 (2023-06-12)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.5.0",
|
||||
"version": "0.4.4",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.5.0",
|
||||
"version": "0.4.4",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
2
Makefile
2
Makefile
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.5.0
|
||||
EXTVERSION = 0.4.4
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.5.0
|
||||
EXTVERSION = 0.4.4
|
||||
|
||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||
HEADERS = src\vector.h
|
||||
|
||||
124
README.md
124
README.md
@@ -18,7 +18,7 @@ Compile and install the extension (supports Postgres 11+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -157,16 +157,7 @@ SELECT category_id, AVG(embedding) FROM items GROUP BY category_id;
|
||||
|
||||
By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
|
||||
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for speed. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Supported index types are:
|
||||
|
||||
- [IVFFlat](#ivfflat)
|
||||
- [HNSW](#hnsw) - added in 0.5.0
|
||||
|
||||
## IVFFlat
|
||||
|
||||
An IVFFlat index divides vectors into lists, and then searches a subset of those lists that are closest to the query vector. It has faster build times and uses less memory than HNSW, but has lower query performance (in terms of speed-recall tradeoff).
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for performance. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Three keys to achieving good recall are:
|
||||
|
||||
@@ -215,63 +206,7 @@ SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## HNSW
|
||||
|
||||
An HNSW index creates a multilayer graph. It has slower build times and uses more memory than IVFFlat, but has better query performance (in terms of speed-recall tradeoff). There’s no training step like IVFFlat, so the index can be created without any data in the table.
|
||||
|
||||
Add an index for each distance function you want to use.
|
||||
|
||||
L2 distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
|
||||
```
|
||||
|
||||
Inner product
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_ip_ops);
|
||||
```
|
||||
|
||||
Cosine distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
|
||||
### Index Options
|
||||
|
||||
Specify HNSW parameters
|
||||
|
||||
- `m` - the max number of connections per layer (16 by default)
|
||||
- `ef_construction` - the size of the dynamic candidate list for constructing the graph (64 by default)
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WITH (m = 16, ef_construction = 64);
|
||||
```
|
||||
|
||||
### Query Options
|
||||
|
||||
Specify the size of the dynamic candidate list for search (40 by default)
|
||||
|
||||
```sql
|
||||
SET hnsw.ef_search = 100;
|
||||
```
|
||||
|
||||
A higher value provides better recall at the cost of speed.
|
||||
|
||||
Use `SET LOCAL` inside a transaction to set it for a single query
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
SET LOCAL hnsw.ef_search = 100;
|
||||
SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## Indexing Progress
|
||||
### Indexing Progress
|
||||
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
|
||||
|
||||
@@ -282,8 +217,8 @@ SELECT phase, tuples_done, tuples_total FROM pg_stat_progress_create_index;
|
||||
The phases are:
|
||||
|
||||
1. `initializing`
|
||||
2. `performing k-means` - IVFFlat only
|
||||
3. `assigning tuples` - IVFFlat only
|
||||
2. `performing k-means`
|
||||
3. `sorting tuples`
|
||||
4. `loading tuples`
|
||||
|
||||
Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
|
||||
@@ -348,7 +283,7 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
|
||||
### Approximate Search
|
||||
|
||||
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
|
||||
To speed up queries with an index, increase the number of inverted lists (at the expense of recall).
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
@@ -424,7 +359,7 @@ or choose to store vectors inline:
|
||||
ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
|
||||
```
|
||||
|
||||
#### Why are there less results for a query after adding an IVFFlat index?
|
||||
#### Why are there less results for a query after adding an index?
|
||||
|
||||
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
||||
|
||||
@@ -440,32 +375,32 @@ Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a singl
|
||||
|
||||
### Vector Operators
|
||||
|
||||
Operator | Description | Added
|
||||
--- | --- | ---
|
||||
\+ | element-wise addition |
|
||||
\- | element-wise subtraction |
|
||||
\* | element-wise multiplication | 0.5.0
|
||||
<-> | Euclidean distance |
|
||||
<#> | negative inner product |
|
||||
<=> | cosine distance |
|
||||
Operator | Description
|
||||
--- | ---
|
||||
\+ | element-wise addition
|
||||
\- | element-wise subtraction
|
||||
\* | element-wise multiplication [unreleased]
|
||||
<-> | Euclidean distance
|
||||
<#> | negative inner product
|
||||
<=> | cosine distance
|
||||
|
||||
### Vector Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
cosine_distance(vector, vector) → double precision | cosine distance |
|
||||
inner_product(vector, vector) → double precision | inner product |
|
||||
l2_distance(vector, vector) → double precision | Euclidean distance |
|
||||
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
|
||||
vector_dims(vector) → integer | number of dimensions |
|
||||
vector_norm(vector) → double precision | Euclidean norm |
|
||||
Function | Description
|
||||
--- | ---
|
||||
cosine_distance(vector, vector) → double precision | cosine distance
|
||||
inner_product(vector, vector) → double precision | inner product
|
||||
l2_distance(vector, vector) → double precision | Euclidean distance
|
||||
l1_distance(vector, vector) → double precision | taxicab distance [unreleased]
|
||||
vector_dims(vector) → integer | number of dimensions
|
||||
vector_norm(vector) → double precision | Euclidean norm
|
||||
|
||||
### Aggregate Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
avg(vector) → vector | average |
|
||||
sum(vector) → vector | sum | 0.5.0
|
||||
Function | Description
|
||||
--- | ---
|
||||
avg(vector) → vector | arithmetic mean
|
||||
sum(vector) → vector | sum [unreleased]
|
||||
|
||||
## Installation Notes
|
||||
|
||||
@@ -509,7 +444,7 @@ Then use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\15"
|
||||
git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
@@ -530,8 +465,9 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
git cherry-pick 237a6df
|
||||
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.5.1'" to load this file. \quit
|
||||
|
||||
CREATE FUNCTION angular_distance(vector, vector) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
@@ -87,9 +87,6 @@ CREATE FUNCTION vector_l2_squared_distance(vector, vector) RETURNS float8
|
||||
CREATE FUNCTION vector_negative_inner_product(vector, vector) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION angular_distance(vector, vector) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_spherical_distance(vector, vector) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
#define HNSW_DEFAULT_M 16
|
||||
#define HNSW_MIN_M 2
|
||||
#define HNSW_MAX_M 100
|
||||
#define HNSW_DEFAULT_EF_CONSTRUCTION 64
|
||||
#define HNSW_DEFAULT_EF_CONSTRUCTION 40
|
||||
#define HNSW_MIN_EF_CONSTRUCTION 4
|
||||
#define HNSW_MAX_EF_CONSTRUCTION 1000
|
||||
#define HNSW_DEFAULT_EF_SEARCH 40
|
||||
|
||||
@@ -185,7 +185,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
scan->xs_ctup.t_self = *tid;
|
||||
#endif
|
||||
|
||||
/* Unpin buffer */
|
||||
if (BufferIsValid(so->buf))
|
||||
ReleaseBuffer(so->buf);
|
||||
|
||||
|
||||
@@ -623,6 +623,10 @@ HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *pro
|
||||
|
||||
Assert(!e->element->deleted);
|
||||
|
||||
/* Skip self for vacuuming update */
|
||||
if (skipElement != NULL && e->element->blkno == skipElement->blkno && e->element->offno == skipElement->offno)
|
||||
continue;
|
||||
|
||||
/* Make robust to issues */
|
||||
if (e->element->level < lc)
|
||||
continue;
|
||||
@@ -909,10 +913,10 @@ HnswUpdateConnection(HnswElement element, HnswCandidate * hc, int m, int lc, int
|
||||
}
|
||||
|
||||
/*
|
||||
* Remove elements being deleted or skipped
|
||||
* Remove elements being deleted
|
||||
*/
|
||||
static List *
|
||||
RemoveElements(List *w, HnswElement skipElement)
|
||||
RemoveElementsBeingDeleted(List *w)
|
||||
{
|
||||
ListCell *lc2;
|
||||
List *w2 = NIL;
|
||||
@@ -921,10 +925,6 @@ RemoveElements(List *w, HnswElement skipElement)
|
||||
{
|
||||
HnswCandidate *hc = (HnswCandidate *) lfirst(lc2);
|
||||
|
||||
/* Skip self for vacuuming update */
|
||||
if (skipElement != NULL && hc->element->blkno == skipElement->blkno && hc->element->offno == skipElement->offno)
|
||||
continue;
|
||||
|
||||
if (list_length(hc->element->heaptids) != 0)
|
||||
w2 = lappend(w2, hc);
|
||||
}
|
||||
@@ -963,27 +963,20 @@ HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, F
|
||||
if (level > entryLevel)
|
||||
level = entryLevel;
|
||||
|
||||
/* Add one for existing element */
|
||||
if (existing)
|
||||
efConstruction++;
|
||||
|
||||
/* 2nd phase */
|
||||
for (int lc = level; lc >= 0; lc--)
|
||||
{
|
||||
int lm = HnswGetLayerM(m, lc);
|
||||
List *neighbors;
|
||||
List *lw;
|
||||
|
||||
w = HnswSearchLayer(q, ep, efConstruction, lc, index, procinfo, collation, true, skipElement);
|
||||
|
||||
/* Elements being deleted or skipped can help with search */
|
||||
/* Elements being deleted can help with search */
|
||||
/* but should be removed before selecting neighbors */
|
||||
if (index != NULL)
|
||||
lw = RemoveElements(w, skipElement);
|
||||
else
|
||||
lw = w;
|
||||
w = RemoveElementsBeingDeleted(w);
|
||||
|
||||
neighbors = SelectNeighbors(lw, lm, lc, procinfo, collation, NULL);
|
||||
neighbors = SelectNeighbors(w, lm, lc, procinfo, collation, NULL);
|
||||
|
||||
AddConnections(element, neighbors, lm, lc);
|
||||
|
||||
|
||||
@@ -611,6 +611,67 @@ FreeVacuumState(HnswVacuumState * vacuumstate)
|
||||
MemoryContextDelete(vacuumstate->tmpCtx);
|
||||
}
|
||||
|
||||
/*
|
||||
* Print graph
|
||||
*/
|
||||
#ifdef HNSW_DEBUG
|
||||
static void
|
||||
PrintGraph(HnswVacuumState * vacuumstate)
|
||||
{
|
||||
BlockNumber blkno = HNSW_HEAD_BLKNO;
|
||||
Relation index = vacuumstate->index;
|
||||
|
||||
while (BlockNumberIsValid(blkno))
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
buf = ReadBuffer(index, blkno);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
maxoffno = PageGetMaxOffsetNumber(page);
|
||||
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
HnswElement element;
|
||||
|
||||
/* Skip neighbor tuples */
|
||||
if (!HnswIsElementTuple(etup))
|
||||
continue;
|
||||
|
||||
/* Skip deleted tuples */
|
||||
if (etup->deleted)
|
||||
continue;
|
||||
|
||||
element = HnswInitElementFromBlock(blkno, offno);
|
||||
HnswLoadElementFromTuple(element, etup, false, true);
|
||||
HnswLoadNeighbors(element, index);
|
||||
|
||||
elog(INFO, "element (%d,%d)", element->blkno, element->offno);
|
||||
|
||||
for (int lc = element->level; lc >= 0; lc--)
|
||||
{
|
||||
HnswNeighborArray *neighbors = &element->neighbors[lc];
|
||||
|
||||
for (int i = 0; i < neighbors->length; i++)
|
||||
{
|
||||
HnswElement e = neighbors->items[i].element;
|
||||
|
||||
elog(INFO, "%d: (%d,%d)", lc, e->blkno, e->offno);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blkno = HnswPageGetOpaque(page)->nextblkno;
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Bulk delete tuples from the index
|
||||
*/
|
||||
@@ -631,6 +692,10 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
|
||||
/* Pass 3: Mark as deleted */
|
||||
MarkDeleted(&vacuumstate);
|
||||
|
||||
#ifdef HNSW_DEBUG
|
||||
PrintGraph(&vacuumstate);
|
||||
#endif
|
||||
|
||||
FreeVacuumState(&vacuumstate);
|
||||
|
||||
return vacuumstate.stats;
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
#include "ivfflat.h"
|
||||
#include "miscadmin.h"
|
||||
#include "storage/bufmgr.h"
|
||||
#include "tcop/tcopprot.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "tcop/tcopprot.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 140000
|
||||
#include "utils/backend_progress.h"
|
||||
|
||||
@@ -343,7 +343,6 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
scan->xs_ctup.t_self = *tid;
|
||||
#endif
|
||||
|
||||
/* Unpin buffer */
|
||||
if (BufferIsValid(so->buf))
|
||||
ReleaseBuffer(so->buf);
|
||||
|
||||
|
||||
47
src/vector.c
47
src/vector.c
@@ -684,49 +684,6 @@ cosine_distance(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_FLOAT8(1.0 - similarity);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the angular distance between two vectors
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(angular_distance);
|
||||
Datum
|
||||
angular_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
float *ax = a->x;
|
||||
float *bx = b->x;
|
||||
float distance = 0.0;
|
||||
float norma = 0.0;
|
||||
float normb = 0.0;
|
||||
double similarity;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
distance += ax[i] * bx[i];
|
||||
norma += ax[i] * ax[i];
|
||||
normb += bx[i] * bx[i];
|
||||
}
|
||||
|
||||
similarity = (double) distance / sqrt((double) norma * (double) normb);
|
||||
|
||||
#ifdef _MSC_VER
|
||||
/* /fp:fast may not propagate NaN */
|
||||
if (isnan(similarity))
|
||||
PG_RETURN_FLOAT8(NAN);
|
||||
#endif
|
||||
|
||||
/* Prevent NaN with acos with loss of precision */
|
||||
if (similarity > 1)
|
||||
similarity = 1;
|
||||
else if (similarity < -1)
|
||||
similarity = -1;
|
||||
|
||||
PG_RETURN_FLOAT8(acos(similarity) / M_PI);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the distance for spherical k-means
|
||||
* Currently uses angular distance since needs to satisfy triangle inequality
|
||||
@@ -738,8 +695,6 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
float *ax = a->x;
|
||||
float *bx = b->x;
|
||||
float dp = 0.0;
|
||||
double distance;
|
||||
|
||||
@@ -747,7 +702,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
dp += ax[i] * bx[i];
|
||||
dp += a->x[i] * b->x[i];
|
||||
|
||||
distance = (double) dp;
|
||||
|
||||
|
||||
@@ -152,56 +152,6 @@ SELECT l1_distance('[3e38]', '[-3e38]');
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,2]', '[2,4]');
|
||||
angular_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,2]', '[0,0]');
|
||||
angular_distance
|
||||
------------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,1]', '[1,1]');
|
||||
angular_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,0]', '[0,2]');
|
||||
angular_distance
|
||||
------------------
|
||||
0.5
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,1]', '[-1,-1]');
|
||||
angular_distance
|
||||
------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT angular_distance('[1,1]', '[1.1,1.1]');
|
||||
angular_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[1,1]', '[-1.1,-1.1]');
|
||||
angular_distance
|
||||
------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT angular_distance('[3e38]', '[3e38]');
|
||||
angular_distance
|
||||
------------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
avg
|
||||
-----------
|
||||
|
||||
@@ -36,16 +36,6 @@ SELECT l1_distance('[0,0]', '[0,1]');
|
||||
SELECT l1_distance('[1,2]', '[3]');
|
||||
SELECT l1_distance('[3e38]', '[-3e38]');
|
||||
|
||||
SELECT angular_distance('[1,2]', '[2,4]');
|
||||
SELECT angular_distance('[1,2]', '[0,0]');
|
||||
SELECT angular_distance('[1,1]', '[1,1]');
|
||||
SELECT angular_distance('[1,0]', '[0,2]');
|
||||
SELECT angular_distance('[1,1]', '[-1,-1]');
|
||||
SELECT angular_distance('[1,2]', '[3]');
|
||||
SELECT angular_distance('[1,1]', '[1.1,1.1]');
|
||||
SELECT angular_distance('[1,1]', '[-1.1,-1.1]');
|
||||
SELECT angular_distance('[3e38]', '[3e38]');
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||
|
||||
@@ -19,6 +19,8 @@ sub test_index_replay
|
||||
|
||||
# Wait for replica to catch up
|
||||
my $applname = $node_replica->name;
|
||||
|
||||
my $server_version_num = $node_primary->safe_psql("postgres", "SHOW server_version_num");
|
||||
my $caughtup_query = "SELECT pg_current_wal_lsn() <= replay_lsn FROM pg_stat_replication WHERE application_name = '$applname';";
|
||||
$node_primary->poll_query_until('postgres', $caughtup_query)
|
||||
or die "Timed out while waiting for replica 1 to catch up";
|
||||
|
||||
@@ -94,7 +94,7 @@ for my $i (0 .. $#operators)
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
# TODO fix test
|
||||
test_recall(1, 0.71, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
@@ -115,7 +115,7 @@ for my $i (0 .. $#operators)
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
# TODO fix test
|
||||
test_recall(1, 0.71, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
@@ -19,6 +19,8 @@ sub test_index_replay
|
||||
|
||||
# Wait for replica to catch up
|
||||
my $applname = $node_replica->name;
|
||||
|
||||
my $server_version_num = $node_primary->safe_psql("postgres", "SHOW server_version_num");
|
||||
my $caughtup_query = "SELECT pg_current_wal_lsn() <= replay_lsn FROM pg_stat_replication WHERE application_name = '$applname';";
|
||||
$node_primary->poll_query_until('postgres', $caughtup_query)
|
||||
or die "Timed out while waiting for replica 1 to catch up";
|
||||
|
||||
@@ -89,7 +89,7 @@ foreach (@queries)
|
||||
test_recall(0.20, $limit, "before vacuum");
|
||||
test_recall(0.95, 100, "before vacuum");
|
||||
|
||||
# TODO Test concurrent inserts with vacuum
|
||||
# TODO test concurrent inserts with vacuum
|
||||
$node->safe_psql("postgres", "VACUUM tst;");
|
||||
|
||||
test_recall(0.95, $limit, "after vacuum");
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($probes, $min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx on tst/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector(3));");
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r1 = rand();
|
||||
my $r2 = rand();
|
||||
my $r3 = rand();
|
||||
push(@queries, "[$r1,$r2,$r3]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Add index
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v $opclass);");
|
||||
|
||||
# Use concurrent inserts
|
||||
$node->pgbench(
|
||||
"--no-vacuum --client=10 --transactions=1000",
|
||||
0,
|
||||
[qr{actually processed}],
|
||||
[qr{^$}],
|
||||
"concurrent INSERTs",
|
||||
{
|
||||
"017_ivfflat_insert_recall_$opclass" => "INSERT INTO tst (v) SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 10) i;"
|
||||
}
|
||||
);
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_indexscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
test_recall(1, 0.71, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
# Account for equal distances
|
||||
test_recall(100, 0.9925, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
$node->safe_psql("postgres", "TRUNCATE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat access method'
|
||||
default_version = '0.5.0'
|
||||
default_version = '0.4.4'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user