Compare commits

..

15 Commits

Author SHA1 Message Date
Andrew Kane
c8e4bf1c3c Added test for arrays 2024-02-03 16:50:17 -08:00
Andrew Kane
2716a223a6 Fixed error with ANALYZE and vectors with different dimensions - fixes #451 2024-02-02 10:47:48 -08:00
Andrew Kane
3697043898 Added note about vacuuming - closes #450 [skip ci] 2024-01-31 14:35:27 -08:00
Andrew Kane
c22740962c Added note for Docker [skip ci] 2024-01-29 15:33:29 -08:00
Andrew Kane
a55ecf3281 Updated changelog [skip ci] 2024-01-29 11:20:33 -08:00
Andrew Kane
fecdd5794e Updated readme [skip ci] 2024-01-29 10:57:54 -08:00
Andrew Kane
281d4fcf60 Version bump to 0.6.0 [skip ci] 2024-01-29 10:54:44 -08:00
Andrew Kane
5b43aaad5a Removed unneeded item ids 2024-01-28 23:14:52 -08:00
Andrew Kane
797ce8034c Ran pgindent [skip ci] 2024-01-28 23:03:48 -08:00
Andrew Kane
cad9e22d9a Updated variable name to be consistent with CreateGraphPages [skip ci] 2024-01-28 23:02:13 -08:00
Andrew Kane
a7d43904f7 Zero memory for each element 2024-01-28 23:00:39 -08:00
Andrew Kane
ded649891b DRY HNSW tuple alloc size [skip ci] 2024-01-28 22:58:38 -08:00
Heikki Linnakangas
2d092016fc Remove unnecessary PageIndexTupleOverwrite calls that caused UB (#438)
These places called PageIndexTupleOverwrite(), with the new tuple
pointing directly to the original page. The PageIndexTupleOverwrite()
call is unnecessary in these cases, as we have already modified the
tuple on the page directly. Moreover, PageIndexTupleOverWrite() will
call memcpy with same src and dst arguments, which is undefined
behavior.

It's OK to modify the pages on disk directly, and we don't need
critical sections, because we either use the generic xlog functions
which create a temporary copy of the page, or we are building a new
index so if we crash the whole index is invisible and will be dropped
anyway.
2024-01-28 22:52:56 -08:00
Andrew Kane
86b31fdf96 Revert "Update neighbor tuples in-place for HNSW index build"
This reverts commit 270dd8189a.
2024-01-28 22:18:22 -08:00
Andrew Kane
5023269f0d Updated changelog [skip ci] 2024-01-28 21:42:30 -08:00
15 changed files with 280 additions and 96 deletions

View File

@@ -1,11 +1,15 @@
## 0.6.0 (unreleased)
## 0.6.1 (unreleased)
If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060-unreleased).
- Fixed error with `ANALYZE` and vectors with different dimensions
## 0.6.0 (2024-01-29)
If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060).
- Changed storage for vector from `extended` to `external`
- Improved performance of HNSW
- Added support for parallel index builds for HNSW
- Added validation for GUC parameters
- Changed storage for vector from `extended` to `external`
- Improved performance of HNSW
- Reduced memory usage for HNSW index builds
- Reduced WAL generation for HNSW index builds
- Fixed error with logical replication

View File

@@ -1,4 +1,4 @@
ARG PG_MAJOR=15
ARG PG_MAJOR=16
FROM postgres:$PG_MAJOR
ARG PG_MAJOR

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.5.1",
"version": "0.6.0",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.5.1",
"version": "0.6.0",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.5.1
EXTVERSION = 0.6.0
MODULE_big = vector
DATA = $(wildcard sql/*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.5.1
EXTVERSION = 0.6.0
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
HEADERS = src\vector.h

View File

@@ -16,11 +16,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
### Linux and Mac
Compile and install the extension (supports Postgres 11+)
Compile and install the extension (supports Postgres 12+)
```sh
cd /tmp
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -44,7 +44,7 @@ Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -264,7 +264,7 @@ HINT: Increase maintenance_work_mem to speed up builds.
Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on the server
Starting with 0.6.0 [unreleased], you can also speed up index creation by increasing the number of parallel workers (2 by default)
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql
SET max_parallel_maintenance_workers = 7; -- plus leader
@@ -430,6 +430,15 @@ To speed up queries with an IVFFlat index, increase the number of inverted lists
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
```
## Vacuuming
Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
```sql
REINDEX INDEX CONCURRENTLY index_name;
VACUUM table_name;
```
## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
@@ -651,20 +660,20 @@ If compilation fails and the output includes `warning: no such sysroot directory
### Docker
Get the [Docker image](https://hub.docker.com/r/ankane/pgvector) with:
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
```sh
docker pull ankane/pgvector
docker pull pgvector/pgvector:pg16
```
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (run it the same way).
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
You can also build the image manually:
```sh
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
```
### Homebrew
@@ -741,7 +750,7 @@ SELECT extversion FROM pg_extension WHERE extname = 'vector';
## Upgrade Notes
### 0.6.0 [unreleased]
### 0.6.0
#### Postgres 12

View File

@@ -60,6 +60,7 @@
#define PROGRESS_HNSW_PHASE_LOAD 2
#define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData))
#define HNSW_TUPLE_ALLOC_SIZE BLCKSZ
#define HNSW_ELEMENT_TUPLE_SIZE(size) MAXALIGN(offsetof(HnswElementTupleData, data) + (size))
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
@@ -354,6 +355,7 @@ typedef struct HnswVacuumState
/* Variables */
struct tidhash_hash *deleted;
BufferAccessStrategy bas;
HnswNeighborTuple ntup;
HnswElementData highestPoint;
/* Memory */

View File

@@ -148,7 +148,6 @@ CreateGraphPages(HnswBuildState * buildstate)
{
Relation index = buildstate->index;
ForkNumber forkNum = buildstate->forkNum;
Size etupAllocSize;
Size maxSize;
HnswElementTuple etup;
HnswNeighborTuple ntup;
@@ -160,12 +159,11 @@ CreateGraphPages(HnswBuildState * buildstate)
char *base = buildstate->hnswarea;
/* Calculate sizes */
etupAllocSize = BLCKSZ;
maxSize = HNSW_MAX_SIZE;
/* Allocate once */
etup = palloc0(etupAllocSize);
ntup = palloc0(BLCKSZ);
etup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
/* Prepare first page */
buf = HnswNewBuffer(index, forkNum);
@@ -184,7 +182,7 @@ CreateGraphPages(HnswBuildState * buildstate)
iter = element->next;
/* Zero memory for each element */
MemSet(etup, 0, etupAllocSize);
MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE);
/* Calculate sizes */
etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(valuePtr));
@@ -192,7 +190,7 @@ CreateGraphPages(HnswBuildState * buildstate)
combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
/* Initial size check */
if (etupSize > etupAllocSize)
if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
elog(ERROR, "index tuple too large");
HnswSetElementTuple(base, etup, element);
@@ -254,38 +252,43 @@ WriteNeighborTuples(HnswBuildState * buildstate)
int m = buildstate->m;
HnswElementPtr iter = buildstate->graph->head;
char *base = buildstate->hnswarea;
HnswNeighborTuple ntup;
/* Allocate once */
ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
while (!HnswPtrIsNull(base, iter))
{
HnswElement e = HnswPtrAccess(base, iter);
HnswElement element = HnswPtrAccess(base, iter);
Buffer buf;
Page page;
ItemId itemid;
HnswNeighborTuple ntup;
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
/* Update iterator */
iter = e->next;
iter = element->next;
/* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
/* Can take a while, so ensure we can interrupt */
/* Needs to be called when no buffer locks are held */
CHECK_FOR_INTERRUPTS();
buf = ReadBufferExtended(index, forkNum, e->neighborPage, RBM_NORMAL, NULL);
buf = ReadBufferExtended(index, forkNum, element->neighborPage, RBM_NORMAL, NULL);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
page = BufferGetPage(buf);
itemid = PageGetItemId(page, e->neighborOffno);
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
/* Check expected size */
Assert(ItemIdGetLength(itemid) == HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m));
HnswSetNeighborTuple(base, ntup, element, m);
/* Update page in-place */
HnswSetNeighborTuple(base, ntup, e, m);
if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Commit */
MarkBufferDirty(buf);
UnlockReleaseBuffer(buf);
}
pfree(ntup);
}
/*
@@ -666,10 +669,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > HNSW_MAX_DIM)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", HNSW_MAX_DIM);

View File

@@ -355,9 +355,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
Buffer buf;
Page page;
GenericXLogState *state;
ItemId itemid;
HnswNeighborTuple ntup;
Size ntupSize;
int idx = -1;
int startIdx;
HnswElement neighborElement = HnswPtrAccess(base, hc->element);
@@ -396,9 +394,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
}
/* Get tuple */
itemid = PageGetItemId(page, offno);
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
ntupSize = ItemIdGetLength(itemid);
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Calculate index for update */
startIdx = (neighborElement->level - lc) * m;
@@ -427,13 +423,9 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
{
ItemPointer indextid = &ntup->indextids[idx];
/* Update neighbor */
/* Update neighbor on the buffer */
ItemPointerSet(indextid, e->blkno, e->offno);
/* Overwrite tuple */
if (!PageIndexTupleOverwrite(page, offno, (Item) ntup, ntupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Commit */
if (building)
MarkBufferDirty(buf);
@@ -457,9 +449,7 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
Buffer buf;
Page page;
GenericXLogState *state;
ItemId itemid;
HnswElementTuple etup;
Size etupSize;
int i;
/* Read page */
@@ -477,9 +467,7 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
}
/* Find space */
itemid = PageGetItemId(page, dup->offno);
etup = (HnswElementTuple) PageGetItem(page, itemid);
etupSize = ItemIdGetLength(itemid);
etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, dup->offno));
for (i = 0; i < HNSW_HEAPTIDS; i++)
{
if (!ItemPointerIsValid(&etup->heaptids[i]))
@@ -495,13 +483,9 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
return false;
}
/* Add heap TID */
/* Add heap TID, modifying the tuple on the page directly */
etup->heaptids[i] = element->heaptids[0];
/* Overwrite tuple */
if (!PageIndexTupleOverwrite(page, dup->offno, (Item) etup, etupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Commit */
if (building)
MarkBufferDirty(buf);

View File

@@ -60,8 +60,7 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
/* Iterate over nodes */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
ItemId itemid = PageGetItemId(page, offno);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
int idx = 0;
bool itemUpdated = false;
@@ -92,15 +91,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (itemUpdated)
{
Size etupSize = ItemIdGetLength(itemid);
/* Mark rest as invalid */
for (int i = idx; i < HNSW_HEAPTIDS; i++)
ItemPointerSetInvalid(&etup->heaptids[i]);
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
updated = true;
}
}
@@ -198,8 +192,8 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
FmgrInfo *procinfo = vacuumstate->procinfo;
Oid collation = vacuumstate->collation;
BufferAccessStrategy bas = vacuumstate->bas;
ItemId itemid;
HnswNeighborTuple ntup;
HnswNeighborTuple ntup = vacuumstate->ntup;
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
char *base = NULL;
/* Skip if element is entry point */
@@ -213,23 +207,23 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
/* Find neighbors for element, skipping itself */
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
/* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
/* Update neighbor tuple */
/* Do this before getting page to minimize locking */
HnswSetNeighborTuple(base, ntup, element, m);
/* Get neighbor page */
buf = ReadBufferExtended(index, MAIN_FORKNUM, element->neighborPage, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
/* Get neighbor tuple */
itemid = PageGetItemId(page, element->neighborOffno);
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
/* Check expected size */
if (ItemIdGetLength(itemid) != HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m))
/* Overwrite tuple */
if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Update page in-place */
HnswSetNeighborTuple(base, ntup, element, m);
/* Commit */
GenericXLogFinish(state);
UnlockReleaseBuffer(buf);
@@ -482,11 +476,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
/* Update element and neighbors together */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
ItemId itemid = PageGetItemId(page, offno);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswNeighborTuple ntup;
Size etupSize;
Size ntupSize;
Buffer nbuf;
Page npage;
BlockNumber neighborPage;
@@ -510,10 +501,6 @@ MarkDeleted(HnswVacuumState * vacuumstate)
if (ItemPointerIsValid(&etup->heaptids[0]))
continue;
/* Calculate sizes */
etupSize = ItemIdGetLength(itemid);
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(etup->level, vacuumstate->m);
/* Get neighbor page */
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
@@ -540,13 +527,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
for (int i = 0; i < ntup->count; i++)
ItemPointerSetInvalid(&ntup->indextids[i]);
/* Overwrite element tuple */
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/* Overwrite neighbor tuple */
if (!PageIndexTupleOverwrite(npage, neighborOffno, (Item) ntup, ntupSize))
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
/*
* We modified the tuples in place, no need to call
* PageIndexTupleOverwrite
*/
/* Commit */
GenericXLogFinish(state);
@@ -591,6 +575,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
vacuumstate->collation = index->rd_indcollation[0];
vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Hnsw vacuum temporary context",
ALLOCSET_DEFAULT_SIZES);
@@ -610,6 +595,7 @@ FreeVacuumState(HnswVacuumState * vacuumstate)
{
tidhash_destroy(vacuumstate->deleted);
FreeAccessStrategy(vacuumstate->bas);
pfree(vacuumstate->ntup);
MemoryContextDelete(vacuumstate->tmpCtx);
}

View File

@@ -866,9 +866,10 @@ vector_mul(PG_FUNCTION_ARGS)
int
vector_cmp_internal(Vector * a, Vector * b)
{
CheckDims(a, b);
int dim = Min(a->dim, b->dim);
for (int i = 0; i < a->dim; i++)
/* Check values before dimensions to be consistent with Postgres arrays */
for (int i = 0; i < dim; i++)
{
if (a->x[i] < b->x[i])
return -1;
@@ -876,6 +877,13 @@ vector_cmp_internal(Vector * a, Vector * b)
if (a->x[i] > b->x[i])
return 1;
}
if (a->dim < b->dim)
return -1;
if (a->dim > b->dim)
return 1;
return 0;
}
@@ -889,6 +897,9 @@ vector_lt(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
}
@@ -902,6 +913,9 @@ vector_le(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
}
@@ -915,6 +929,9 @@ vector_eq(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
}
@@ -928,6 +945,9 @@ vector_ne(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
}
@@ -941,6 +961,9 @@ vector_ge(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
}
@@ -954,6 +977,9 @@ vector_gt(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
}

View File

@@ -24,6 +24,56 @@ SELECT '[1e37]'::vector * '[1e37]';
ERROR: value out of range: overflow
SELECT '[1e-37]'::vector * '[1e-37]';
ERROR: value out of range: underflow
SELECT '[1,2,3]'::vector = '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector = '[1,2]';
ERROR: different vector dimensions 3 and 2
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
vector_cmp
------------
0
(1 row)
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[1,2]', '[2,3,4]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[2,3]', '[1,2,3]');
vector_cmp
------------
1
(1 row)
SELECT vector_dims('[1,2,3]');
vector_dims
-------------

View File

@@ -6,6 +6,17 @@ SELECT '[1,2,3]'::vector * '[4,5,6]';
SELECT '[1e37]'::vector * '[1e37]';
SELECT '[1e-37]'::vector * '[1e-37]';
SELECT '[1,2,3]'::vector = '[1,2,3]';
SELECT '[1,2,3]'::vector = '[1,2]';
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
SELECT vector_cmp('[1,2]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[1,2]');
SELECT vector_cmp('[1,2]', '[2,3,4]');
SELECT vector_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);

113
test/t/020_hnsw_array.pl Normal file
View File

@@ -0,0 +1,113 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More;
my $node;
my @queries = ();
my @expected;
my $limit = 20;
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids;
my @expected_ids = split("\n", $expected[$i]);
foreach (@expected_ids)
{
if (exists($actual_set{$_}))
{
$correct++;
}
$total++;
}
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v float4[3]);");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
);
$node->safe_psql("postgres", qq(
CREATE FUNCTION float4_l2_distance(float4[], float4[]) RETURNS float8
AS 'BEGIN RETURN l2_distance(\$1::vector, \$2::vector); END;'
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION float4_l2_squared_distance(float4[], float4[]) RETURNS float8
AS 'BEGIN RETURN vector_l2_squared_distance(\$1::vector, \$2::vector); END;'
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <-> (
LEFTARG = float4[], RIGHTARG = float4[], PROCEDURE = float4_l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR CLASS float4_l2_ops
FOR TYPE float4[] USING hnsw AS
OPERATOR 1 <-> (float4[], float4[]) FOR ORDER BY float_ops,
FUNCTION 1 float4_l2_squared_distance(float4[], float4[]);
));
# Generate queries
for (1 .. 20)
{
my $r1 = rand();
my $r2 = rand();
my $r3 = rand();
push(@queries, "{$r1,$r2,$r3}");
}
# Check each index type
my @operators = ("<->");
my @opclasses = ("float4_l2_ops");
for my $i (0 .. $#operators)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Get exact results
@expected = ();
foreach (@queries)
{
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
push(@expected, $res);
}
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v $opclass);");
my $min = $operator eq "<#>" ? 0.80 : 0.99;
test_recall($min, $operator);
}
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.5.1'
default_version = '0.6.0'
module_pathname = '$libdir/vector'
relocatable = true