mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 12:07:34 +08:00
Compare commits
15 Commits
hnsw-vacuu
...
hnsw-datum
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8e4bf1c3c | ||
|
|
2716a223a6 | ||
|
|
3697043898 | ||
|
|
c22740962c | ||
|
|
a55ecf3281 | ||
|
|
fecdd5794e | ||
|
|
281d4fcf60 | ||
|
|
5b43aaad5a | ||
|
|
797ce8034c | ||
|
|
cad9e22d9a | ||
|
|
a7d43904f7 | ||
|
|
ded649891b | ||
|
|
2d092016fc | ||
|
|
86b31fdf96 | ||
|
|
5023269f0d |
12
CHANGELOG.md
12
CHANGELOG.md
@@ -1,11 +1,15 @@
|
||||
## 0.6.0 (unreleased)
|
||||
## 0.6.1 (unreleased)
|
||||
|
||||
If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060-unreleased).
|
||||
- Fixed error with `ANALYZE` and vectors with different dimensions
|
||||
|
||||
## 0.6.0 (2024-01-29)
|
||||
|
||||
If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060).
|
||||
|
||||
- Changed storage for vector from `extended` to `external`
|
||||
- Improved performance of HNSW
|
||||
- Added support for parallel index builds for HNSW
|
||||
- Added validation for GUC parameters
|
||||
- Changed storage for vector from `extended` to `external`
|
||||
- Improved performance of HNSW
|
||||
- Reduced memory usage for HNSW index builds
|
||||
- Reduced WAL generation for HNSW index builds
|
||||
- Fixed error with logical replication
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG PG_MAJOR=15
|
||||
ARG PG_MAJOR=16
|
||||
FROM postgres:$PG_MAJOR
|
||||
ARG PG_MAJOR
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.5.1",
|
||||
"version": "0.6.0",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.5.1",
|
||||
"version": "0.6.0",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
2
Makefile
2
Makefile
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.5.1
|
||||
EXTVERSION = 0.6.0
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.5.1
|
||||
EXTVERSION = 0.6.0
|
||||
|
||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||
HEADERS = src\vector.h
|
||||
|
||||
29
README.md
29
README.md
@@ -16,11 +16,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
|
||||
|
||||
### Linux and Mac
|
||||
|
||||
Compile and install the extension (supports Postgres 11+)
|
||||
Compile and install the extension (supports Postgres 12+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -44,7 +44,7 @@ Then use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\16"
|
||||
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
@@ -264,7 +264,7 @@ HINT: Increase maintenance_work_mem to speed up builds.
|
||||
|
||||
Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on the server
|
||||
|
||||
Starting with 0.6.0 [unreleased], you can also speed up index creation by increasing the number of parallel workers (2 by default)
|
||||
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
|
||||
|
||||
```sql
|
||||
SET max_parallel_maintenance_workers = 7; -- plus leader
|
||||
@@ -430,6 +430,15 @@ To speed up queries with an IVFFlat index, increase the number of inverted lists
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
```
|
||||
|
||||
## Vacuuming
|
||||
|
||||
Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
|
||||
|
||||
```sql
|
||||
REINDEX INDEX CONCURRENTLY index_name;
|
||||
VACUUM table_name;
|
||||
```
|
||||
|
||||
## Languages
|
||||
|
||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||
@@ -651,20 +660,20 @@ If compilation fails and the output includes `warning: no such sysroot directory
|
||||
|
||||
### Docker
|
||||
|
||||
Get the [Docker image](https://hub.docker.com/r/ankane/pgvector) with:
|
||||
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
|
||||
|
||||
```sh
|
||||
docker pull ankane/pgvector
|
||||
docker pull pgvector/pgvector:pg16
|
||||
```
|
||||
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (run it the same way).
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
|
||||
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
|
||||
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
### Homebrew
|
||||
@@ -741,7 +750,7 @@ SELECT extversion FROM pg_extension WHERE extname = 'vector';
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
### 0.6.0 [unreleased]
|
||||
### 0.6.0
|
||||
|
||||
#### Postgres 12
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
#define PROGRESS_HNSW_PHASE_LOAD 2
|
||||
|
||||
#define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData))
|
||||
#define HNSW_TUPLE_ALLOC_SIZE BLCKSZ
|
||||
|
||||
#define HNSW_ELEMENT_TUPLE_SIZE(size) MAXALIGN(offsetof(HnswElementTupleData, data) + (size))
|
||||
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
|
||||
@@ -354,6 +355,7 @@ typedef struct HnswVacuumState
|
||||
/* Variables */
|
||||
struct tidhash_hash *deleted;
|
||||
BufferAccessStrategy bas;
|
||||
HnswNeighborTuple ntup;
|
||||
HnswElementData highestPoint;
|
||||
|
||||
/* Memory */
|
||||
|
||||
@@ -148,7 +148,6 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
{
|
||||
Relation index = buildstate->index;
|
||||
ForkNumber forkNum = buildstate->forkNum;
|
||||
Size etupAllocSize;
|
||||
Size maxSize;
|
||||
HnswElementTuple etup;
|
||||
HnswNeighborTuple ntup;
|
||||
@@ -160,12 +159,11 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
char *base = buildstate->hnswarea;
|
||||
|
||||
/* Calculate sizes */
|
||||
etupAllocSize = BLCKSZ;
|
||||
maxSize = HNSW_MAX_SIZE;
|
||||
|
||||
/* Allocate once */
|
||||
etup = palloc0(etupAllocSize);
|
||||
ntup = palloc0(BLCKSZ);
|
||||
etup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
/* Prepare first page */
|
||||
buf = HnswNewBuffer(index, forkNum);
|
||||
@@ -184,7 +182,7 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
iter = element->next;
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(etup, 0, etupAllocSize);
|
||||
MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
/* Calculate sizes */
|
||||
etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(valuePtr));
|
||||
@@ -192,7 +190,7 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
|
||||
|
||||
/* Initial size check */
|
||||
if (etupSize > etupAllocSize)
|
||||
if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
|
||||
elog(ERROR, "index tuple too large");
|
||||
|
||||
HnswSetElementTuple(base, etup, element);
|
||||
@@ -254,38 +252,43 @@ WriteNeighborTuples(HnswBuildState * buildstate)
|
||||
int m = buildstate->m;
|
||||
HnswElementPtr iter = buildstate->graph->head;
|
||||
char *base = buildstate->hnswarea;
|
||||
HnswNeighborTuple ntup;
|
||||
|
||||
/* Allocate once */
|
||||
ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
while (!HnswPtrIsNull(base, iter))
|
||||
{
|
||||
HnswElement e = HnswPtrAccess(base, iter);
|
||||
HnswElement element = HnswPtrAccess(base, iter);
|
||||
Buffer buf;
|
||||
Page page;
|
||||
ItemId itemid;
|
||||
HnswNeighborTuple ntup;
|
||||
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
|
||||
|
||||
/* Update iterator */
|
||||
iter = e->next;
|
||||
iter = element->next;
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
/* Can take a while, so ensure we can interrupt */
|
||||
/* Needs to be called when no buffer locks are held */
|
||||
CHECK_FOR_INTERRUPTS();
|
||||
|
||||
buf = ReadBufferExtended(index, forkNum, e->neighborPage, RBM_NORMAL, NULL);
|
||||
buf = ReadBufferExtended(index, forkNum, element->neighborPage, RBM_NORMAL, NULL);
|
||||
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
|
||||
page = BufferGetPage(buf);
|
||||
itemid = PageGetItemId(page, e->neighborOffno);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
|
||||
|
||||
/* Check expected size */
|
||||
Assert(ItemIdGetLength(itemid) == HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m));
|
||||
HnswSetNeighborTuple(base, ntup, element, m);
|
||||
|
||||
/* Update page in-place */
|
||||
HnswSetNeighborTuple(base, ntup, e, m);
|
||||
if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
|
||||
pfree(ntup);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -666,10 +669,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
||||
buildstate->efConstruction = HnswGetEfConstruction(index);
|
||||
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
|
||||
|
||||
/* Require column to have dimensions to be indexed */
|
||||
if (buildstate->dimensions < 0)
|
||||
elog(ERROR, "column does not have dimensions");
|
||||
|
||||
if (buildstate->dimensions > HNSW_MAX_DIM)
|
||||
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", HNSW_MAX_DIM);
|
||||
|
||||
|
||||
@@ -355,9 +355,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
|
||||
Buffer buf;
|
||||
Page page;
|
||||
GenericXLogState *state;
|
||||
ItemId itemid;
|
||||
HnswNeighborTuple ntup;
|
||||
Size ntupSize;
|
||||
int idx = -1;
|
||||
int startIdx;
|
||||
HnswElement neighborElement = HnswPtrAccess(base, hc->element);
|
||||
@@ -396,9 +394,7 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
|
||||
}
|
||||
|
||||
/* Get tuple */
|
||||
itemid = PageGetItemId(page, offno);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
|
||||
ntupSize = ItemIdGetLength(itemid);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
|
||||
/* Calculate index for update */
|
||||
startIdx = (neighborElement->level - lc) * m;
|
||||
@@ -427,13 +423,9 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
|
||||
{
|
||||
ItemPointer indextid = &ntup->indextids[idx];
|
||||
|
||||
/* Update neighbor */
|
||||
/* Update neighbor on the buffer */
|
||||
ItemPointerSet(indextid, e->blkno, e->offno);
|
||||
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
if (building)
|
||||
MarkBufferDirty(buf);
|
||||
@@ -457,9 +449,7 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
|
||||
Buffer buf;
|
||||
Page page;
|
||||
GenericXLogState *state;
|
||||
ItemId itemid;
|
||||
HnswElementTuple etup;
|
||||
Size etupSize;
|
||||
int i;
|
||||
|
||||
/* Read page */
|
||||
@@ -477,9 +467,7 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
|
||||
}
|
||||
|
||||
/* Find space */
|
||||
itemid = PageGetItemId(page, dup->offno);
|
||||
etup = (HnswElementTuple) PageGetItem(page, itemid);
|
||||
etupSize = ItemIdGetLength(itemid);
|
||||
etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, dup->offno));
|
||||
for (i = 0; i < HNSW_HEAPTIDS; i++)
|
||||
{
|
||||
if (!ItemPointerIsValid(&etup->heaptids[i]))
|
||||
@@ -495,13 +483,9 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Add heap TID */
|
||||
/* Add heap TID, modifying the tuple on the page directly */
|
||||
etup->heaptids[i] = element->heaptids[0];
|
||||
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, dup->offno, (Item) etup, etupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
if (building)
|
||||
MarkBufferDirty(buf);
|
||||
|
||||
@@ -60,8 +60,7 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
|
||||
/* Iterate over nodes */
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
ItemId itemid = PageGetItemId(page, offno);
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
int idx = 0;
|
||||
bool itemUpdated = false;
|
||||
|
||||
@@ -92,15 +91,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
|
||||
|
||||
if (itemUpdated)
|
||||
{
|
||||
Size etupSize = ItemIdGetLength(itemid);
|
||||
|
||||
/* Mark rest as invalid */
|
||||
for (int i = idx; i < HNSW_HEAPTIDS; i++)
|
||||
ItemPointerSetInvalid(&etup->heaptids[i]);
|
||||
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
updated = true;
|
||||
}
|
||||
}
|
||||
@@ -198,8 +192,8 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
|
||||
FmgrInfo *procinfo = vacuumstate->procinfo;
|
||||
Oid collation = vacuumstate->collation;
|
||||
BufferAccessStrategy bas = vacuumstate->bas;
|
||||
ItemId itemid;
|
||||
HnswNeighborTuple ntup;
|
||||
HnswNeighborTuple ntup = vacuumstate->ntup;
|
||||
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
|
||||
char *base = NULL;
|
||||
|
||||
/* Skip if element is entry point */
|
||||
@@ -213,23 +207,23 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
|
||||
/* Find neighbors for element, skipping itself */
|
||||
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
/* Update neighbor tuple */
|
||||
/* Do this before getting page to minimize locking */
|
||||
HnswSetNeighborTuple(base, ntup, element, m);
|
||||
|
||||
/* Get neighbor page */
|
||||
buf = ReadBufferExtended(index, MAIN_FORKNUM, element->neighborPage, RBM_NORMAL, bas);
|
||||
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
|
||||
state = GenericXLogStart(index);
|
||||
page = GenericXLogRegisterBuffer(state, buf, 0);
|
||||
|
||||
/* Get neighbor tuple */
|
||||
itemid = PageGetItemId(page, element->neighborOffno);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
|
||||
|
||||
/* Check expected size */
|
||||
if (ItemIdGetLength(itemid) != HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m))
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Update page in-place */
|
||||
HnswSetNeighborTuple(base, ntup, element, m);
|
||||
|
||||
/* Commit */
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
@@ -482,11 +476,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
/* Update element and neighbors together */
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
ItemId itemid = PageGetItemId(page, offno);
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
HnswNeighborTuple ntup;
|
||||
Size etupSize;
|
||||
Size ntupSize;
|
||||
Buffer nbuf;
|
||||
Page npage;
|
||||
BlockNumber neighborPage;
|
||||
@@ -510,10 +501,6 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
if (ItemPointerIsValid(&etup->heaptids[0]))
|
||||
continue;
|
||||
|
||||
/* Calculate sizes */
|
||||
etupSize = ItemIdGetLength(itemid);
|
||||
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(etup->level, vacuumstate->m);
|
||||
|
||||
/* Get neighbor page */
|
||||
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
|
||||
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
|
||||
@@ -540,13 +527,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
for (int i = 0; i < ntup->count; i++)
|
||||
ItemPointerSetInvalid(&ntup->indextids[i]);
|
||||
|
||||
/* Overwrite element tuple */
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Overwrite neighbor tuple */
|
||||
if (!PageIndexTupleOverwrite(npage, neighborOffno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
/*
|
||||
* We modified the tuples in place, no need to call
|
||||
* PageIndexTupleOverwrite
|
||||
*/
|
||||
|
||||
/* Commit */
|
||||
GenericXLogFinish(state);
|
||||
@@ -591,6 +575,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
|
||||
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
|
||||
vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
|
||||
vacuumstate->collation = index->rd_indcollation[0];
|
||||
vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
|
||||
"Hnsw vacuum temporary context",
|
||||
ALLOCSET_DEFAULT_SIZES);
|
||||
@@ -610,6 +595,7 @@ FreeVacuumState(HnswVacuumState * vacuumstate)
|
||||
{
|
||||
tidhash_destroy(vacuumstate->deleted);
|
||||
FreeAccessStrategy(vacuumstate->bas);
|
||||
pfree(vacuumstate->ntup);
|
||||
MemoryContextDelete(vacuumstate->tmpCtx);
|
||||
}
|
||||
|
||||
|
||||
30
src/vector.c
30
src/vector.c
@@ -866,9 +866,10 @@ vector_mul(PG_FUNCTION_ARGS)
|
||||
int
|
||||
vector_cmp_internal(Vector * a, Vector * b)
|
||||
{
|
||||
CheckDims(a, b);
|
||||
int dim = Min(a->dim, b->dim);
|
||||
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
/* Check values before dimensions to be consistent with Postgres arrays */
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
if (a->x[i] < b->x[i])
|
||||
return -1;
|
||||
@@ -876,6 +877,13 @@ vector_cmp_internal(Vector * a, Vector * b)
|
||||
if (a->x[i] > b->x[i])
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (a->dim < b->dim)
|
||||
return -1;
|
||||
|
||||
if (a->dim > b->dim)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -889,6 +897,9 @@ vector_lt(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
|
||||
}
|
||||
|
||||
@@ -902,6 +913,9 @@ vector_le(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
|
||||
}
|
||||
|
||||
@@ -915,6 +929,9 @@ vector_eq(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
|
||||
}
|
||||
|
||||
@@ -928,6 +945,9 @@ vector_ne(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
|
||||
}
|
||||
|
||||
@@ -941,6 +961,9 @@ vector_ge(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
|
||||
}
|
||||
|
||||
@@ -954,6 +977,9 @@ vector_gt(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,56 @@ SELECT '[1e37]'::vector * '[1e37]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[1e-37]'::vector * '[1e-37]';
|
||||
ERROR: value out of range: underflow
|
||||
SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector = '[1,2]';
|
||||
ERROR: different vector dimensions 3 and 2
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
|
||||
vector_cmp
|
||||
------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2]');
|
||||
vector_cmp
|
||||
------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2]', '[2,3,4]');
|
||||
vector_cmp
|
||||
------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[2,3]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_dims('[1,2,3]');
|
||||
vector_dims
|
||||
-------------
|
||||
|
||||
@@ -6,6 +6,17 @@ SELECT '[1,2,3]'::vector * '[4,5,6]';
|
||||
SELECT '[1e37]'::vector * '[1e37]';
|
||||
SELECT '[1e-37]'::vector * '[1e-37]';
|
||||
|
||||
SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector = '[1,2]';
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
|
||||
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2]');
|
||||
SELECT vector_cmp('[1,2]', '[2,3,4]');
|
||||
SELECT vector_cmp('[2,3]', '[1,2,3]');
|
||||
|
||||
SELECT vector_dims('[1,2,3]');
|
||||
|
||||
SELECT round(vector_norm('[1,1]')::numeric, 5);
|
||||
|
||||
113
test/t/020_hnsw_array.pl
Normal file
113
test/t/020_hnsw_array.pl
Normal file
@@ -0,0 +1,113 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v float4[3]);");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
|
||||
$node->safe_psql("postgres", qq(
|
||||
CREATE FUNCTION float4_l2_distance(float4[], float4[]) RETURNS float8
|
||||
AS 'BEGIN RETURN l2_distance(\$1::vector, \$2::vector); END;'
|
||||
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION float4_l2_squared_distance(float4[], float4[]) RETURNS float8
|
||||
AS 'BEGIN RETURN vector_l2_squared_distance(\$1::vector, \$2::vector); END;'
|
||||
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = float4[], RIGHTARG = float4[], PROCEDURE = float4_l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS float4_l2_ops
|
||||
FOR TYPE float4[] USING hnsw AS
|
||||
OPERATOR 1 <-> (float4[], float4[]) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 float4_l2_squared_distance(float4[], float4[]);
|
||||
));
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r1 = rand();
|
||||
my $r2 = rand();
|
||||
my $r3 = rand();
|
||||
push(@queries, "{$r1,$r2,$r3}");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->");
|
||||
my @opclasses = ("float4_l2_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Add index
|
||||
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v $opclass);");
|
||||
|
||||
my $min = $operator eq "<#>" ? 0.80 : 0.99;
|
||||
test_recall($min, $operator);
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat and hnsw access methods'
|
||||
default_version = '0.5.1'
|
||||
default_version = '0.6.0'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user