mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-23 20:40:55 +08:00
Compare commits
3 Commits
hnsw-datum
...
hnsw-vacuu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3f75886bb2 | ||
|
|
ebf7de0c16 | ||
|
|
142a32af96 |
12
CHANGELOG.md
12
CHANGELOG.md
@@ -1,15 +1,11 @@
|
||||
## 0.6.1 (unreleased)
|
||||
## 0.6.0 (unreleased)
|
||||
|
||||
- Fixed error with `ANALYZE` and vectors with different dimensions
|
||||
If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060-unreleased).
|
||||
|
||||
## 0.6.0 (2024-01-29)
|
||||
|
||||
If upgrading with Postgres 12 or Docker, see [these notes](https://github.com/pgvector/pgvector#060).
|
||||
|
||||
- Added support for parallel index builds for HNSW
|
||||
- Added validation for GUC parameters
|
||||
- Changed storage for vector from `extended` to `external`
|
||||
- Improved performance of HNSW
|
||||
- Added support for parallel index builds for HNSW
|
||||
- Added validation for GUC parameters
|
||||
- Reduced memory usage for HNSW index builds
|
||||
- Reduced WAL generation for HNSW index builds
|
||||
- Fixed error with logical replication
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG PG_MAJOR=16
|
||||
ARG PG_MAJOR=15
|
||||
FROM postgres:$PG_MAJOR
|
||||
ARG PG_MAJOR
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.6.0",
|
||||
"version": "0.5.1",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.6.0",
|
||||
"version": "0.5.1",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
2
Makefile
2
Makefile
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.6.0
|
||||
EXTVERSION = 0.5.1
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.6.0
|
||||
EXTVERSION = 0.5.1
|
||||
|
||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||
HEADERS = src\vector.h
|
||||
|
||||
29
README.md
29
README.md
@@ -16,11 +16,11 @@ Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recove
|
||||
|
||||
### Linux and Mac
|
||||
|
||||
Compile and install the extension (supports Postgres 12+)
|
||||
Compile and install the extension (supports Postgres 11+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -44,7 +44,7 @@ Then use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\16"
|
||||
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
@@ -264,7 +264,7 @@ HINT: Increase maintenance_work_mem to speed up builds.
|
||||
|
||||
Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on the server
|
||||
|
||||
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
|
||||
Starting with 0.6.0 [unreleased], you can also speed up index creation by increasing the number of parallel workers (2 by default)
|
||||
|
||||
```sql
|
||||
SET max_parallel_maintenance_workers = 7; -- plus leader
|
||||
@@ -430,15 +430,6 @@ To speed up queries with an IVFFlat index, increase the number of inverted lists
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
```
|
||||
|
||||
## Vacuuming
|
||||
|
||||
Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
|
||||
|
||||
```sql
|
||||
REINDEX INDEX CONCURRENTLY index_name;
|
||||
VACUUM table_name;
|
||||
```
|
||||
|
||||
## Languages
|
||||
|
||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||
@@ -660,20 +651,20 @@ If compilation fails and the output includes `warning: no such sysroot directory
|
||||
|
||||
### Docker
|
||||
|
||||
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
|
||||
Get the [Docker image](https://hub.docker.com/r/ankane/pgvector) with:
|
||||
|
||||
```sh
|
||||
docker pull pgvector/pgvector:pg16
|
||||
docker pull ankane/pgvector
|
||||
```
|
||||
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (run it the same way).
|
||||
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
|
||||
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
### Homebrew
|
||||
@@ -750,7 +741,7 @@ SELECT extversion FROM pg_extension WHERE extname = 'vector';
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
### 0.6.0
|
||||
### 0.6.0 [unreleased]
|
||||
|
||||
#### Postgres 12
|
||||
|
||||
|
||||
@@ -60,7 +60,6 @@
|
||||
#define PROGRESS_HNSW_PHASE_LOAD 2
|
||||
|
||||
#define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData))
|
||||
#define HNSW_TUPLE_ALLOC_SIZE BLCKSZ
|
||||
|
||||
#define HNSW_ELEMENT_TUPLE_SIZE(size) MAXALIGN(offsetof(HnswElementTupleData, data) + (size))
|
||||
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
|
||||
@@ -355,7 +354,6 @@ typedef struct HnswVacuumState
|
||||
/* Variables */
|
||||
struct tidhash_hash *deleted;
|
||||
BufferAccessStrategy bas;
|
||||
HnswNeighborTuple ntup;
|
||||
HnswElementData highestPoint;
|
||||
|
||||
/* Memory */
|
||||
|
||||
@@ -148,6 +148,7 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
{
|
||||
Relation index = buildstate->index;
|
||||
ForkNumber forkNum = buildstate->forkNum;
|
||||
Size etupAllocSize;
|
||||
Size maxSize;
|
||||
HnswElementTuple etup;
|
||||
HnswNeighborTuple ntup;
|
||||
@@ -159,11 +160,12 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
char *base = buildstate->hnswarea;
|
||||
|
||||
/* Calculate sizes */
|
||||
etupAllocSize = BLCKSZ;
|
||||
maxSize = HNSW_MAX_SIZE;
|
||||
|
||||
/* Allocate once */
|
||||
etup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
etup = palloc0(etupAllocSize);
|
||||
ntup = palloc0(BLCKSZ);
|
||||
|
||||
/* Prepare first page */
|
||||
buf = HnswNewBuffer(index, forkNum);
|
||||
@@ -182,7 +184,7 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
iter = element->next;
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
MemSet(etup, 0, etupAllocSize);
|
||||
|
||||
/* Calculate sizes */
|
||||
etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(valuePtr));
|
||||
@@ -190,7 +192,7 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
|
||||
|
||||
/* Initial size check */
|
||||
if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
|
||||
if (etupSize > etupAllocSize)
|
||||
elog(ERROR, "index tuple too large");
|
||||
|
||||
HnswSetElementTuple(base, etup, element);
|
||||
@@ -252,43 +254,38 @@ WriteNeighborTuples(HnswBuildState * buildstate)
|
||||
int m = buildstate->m;
|
||||
HnswElementPtr iter = buildstate->graph->head;
|
||||
char *base = buildstate->hnswarea;
|
||||
HnswNeighborTuple ntup;
|
||||
|
||||
/* Allocate once */
|
||||
ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
while (!HnswPtrIsNull(base, iter))
|
||||
{
|
||||
HnswElement element = HnswPtrAccess(base, iter);
|
||||
HnswElement e = HnswPtrAccess(base, iter);
|
||||
Buffer buf;
|
||||
Page page;
|
||||
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
|
||||
ItemId itemid;
|
||||
HnswNeighborTuple ntup;
|
||||
|
||||
/* Update iterator */
|
||||
iter = element->next;
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
iter = e->next;
|
||||
|
||||
/* Can take a while, so ensure we can interrupt */
|
||||
/* Needs to be called when no buffer locks are held */
|
||||
CHECK_FOR_INTERRUPTS();
|
||||
|
||||
buf = ReadBufferExtended(index, forkNum, element->neighborPage, RBM_NORMAL, NULL);
|
||||
buf = ReadBufferExtended(index, forkNum, e->neighborPage, RBM_NORMAL, NULL);
|
||||
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
|
||||
page = BufferGetPage(buf);
|
||||
itemid = PageGetItemId(page, e->neighborOffno);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
|
||||
|
||||
HnswSetNeighborTuple(base, ntup, element, m);
|
||||
/* Check expected size */
|
||||
Assert(ItemIdGetLength(itemid) == HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m));
|
||||
|
||||
if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
/* Update page in-place */
|
||||
HnswSetNeighborTuple(base, ntup, e, m);
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
|
||||
pfree(ntup);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -669,6 +666,10 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
||||
buildstate->efConstruction = HnswGetEfConstruction(index);
|
||||
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
|
||||
|
||||
/* Require column to have dimensions to be indexed */
|
||||
if (buildstate->dimensions < 0)
|
||||
elog(ERROR, "column does not have dimensions");
|
||||
|
||||
if (buildstate->dimensions > HNSW_MAX_DIM)
|
||||
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", HNSW_MAX_DIM);
|
||||
|
||||
|
||||
@@ -355,7 +355,9 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
|
||||
Buffer buf;
|
||||
Page page;
|
||||
GenericXLogState *state;
|
||||
ItemId itemid;
|
||||
HnswNeighborTuple ntup;
|
||||
Size ntupSize;
|
||||
int idx = -1;
|
||||
int startIdx;
|
||||
HnswElement neighborElement = HnswPtrAccess(base, hc->element);
|
||||
@@ -394,7 +396,9 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
|
||||
}
|
||||
|
||||
/* Get tuple */
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
itemid = PageGetItemId(page, offno);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
|
||||
ntupSize = ItemIdGetLength(itemid);
|
||||
|
||||
/* Calculate index for update */
|
||||
startIdx = (neighborElement->level - lc) * m;
|
||||
@@ -423,9 +427,13 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
|
||||
{
|
||||
ItemPointer indextid = &ntup->indextids[idx];
|
||||
|
||||
/* Update neighbor on the buffer */
|
||||
/* Update neighbor */
|
||||
ItemPointerSet(indextid, e->blkno, e->offno);
|
||||
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
if (building)
|
||||
MarkBufferDirty(buf);
|
||||
@@ -449,7 +457,9 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
|
||||
Buffer buf;
|
||||
Page page;
|
||||
GenericXLogState *state;
|
||||
ItemId itemid;
|
||||
HnswElementTuple etup;
|
||||
Size etupSize;
|
||||
int i;
|
||||
|
||||
/* Read page */
|
||||
@@ -467,7 +477,9 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
|
||||
}
|
||||
|
||||
/* Find space */
|
||||
etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, dup->offno));
|
||||
itemid = PageGetItemId(page, dup->offno);
|
||||
etup = (HnswElementTuple) PageGetItem(page, itemid);
|
||||
etupSize = ItemIdGetLength(itemid);
|
||||
for (i = 0; i < HNSW_HEAPTIDS; i++)
|
||||
{
|
||||
if (!ItemPointerIsValid(&etup->heaptids[i]))
|
||||
@@ -483,9 +495,13 @@ AddDuplicateOnDisk(Relation index, HnswElement element, HnswElement dup, bool bu
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Add heap TID, modifying the tuple on the page directly */
|
||||
/* Add heap TID */
|
||||
etup->heaptids[i] = element->heaptids[0];
|
||||
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, dup->offno, (Item) etup, etupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
if (building)
|
||||
MarkBufferDirty(buf);
|
||||
|
||||
@@ -60,7 +60,8 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
|
||||
/* Iterate over nodes */
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
ItemId itemid = PageGetItemId(page, offno);
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
|
||||
int idx = 0;
|
||||
bool itemUpdated = false;
|
||||
|
||||
@@ -91,10 +92,15 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
|
||||
|
||||
if (itemUpdated)
|
||||
{
|
||||
Size etupSize = ItemIdGetLength(itemid);
|
||||
|
||||
/* Mark rest as invalid */
|
||||
for (int i = idx; i < HNSW_HEAPTIDS; i++)
|
||||
ItemPointerSetInvalid(&etup->heaptids[i]);
|
||||
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
updated = true;
|
||||
}
|
||||
}
|
||||
@@ -192,8 +198,8 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
|
||||
FmgrInfo *procinfo = vacuumstate->procinfo;
|
||||
Oid collation = vacuumstate->collation;
|
||||
BufferAccessStrategy bas = vacuumstate->bas;
|
||||
HnswNeighborTuple ntup = vacuumstate->ntup;
|
||||
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
|
||||
ItemId itemid;
|
||||
HnswNeighborTuple ntup;
|
||||
char *base = NULL;
|
||||
|
||||
/* Skip if element is entry point */
|
||||
@@ -207,23 +213,23 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
|
||||
/* Find neighbors for element, skipping itself */
|
||||
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
|
||||
|
||||
/* Zero memory for each element */
|
||||
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
|
||||
|
||||
/* Update neighbor tuple */
|
||||
/* Do this before getting page to minimize locking */
|
||||
HnswSetNeighborTuple(base, ntup, element, m);
|
||||
|
||||
/* Get neighbor page */
|
||||
buf = ReadBufferExtended(index, MAIN_FORKNUM, element->neighborPage, RBM_NORMAL, bas);
|
||||
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
|
||||
state = GenericXLogStart(index);
|
||||
page = GenericXLogRegisterBuffer(state, buf, 0);
|
||||
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, element->neighborOffno, (Item) ntup, ntupSize))
|
||||
/* Get neighbor tuple */
|
||||
itemid = PageGetItemId(page, element->neighborOffno);
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, itemid);
|
||||
|
||||
/* Check expected size */
|
||||
if (ItemIdGetLength(itemid) != HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Update page in-place */
|
||||
HnswSetNeighborTuple(base, ntup, element, m);
|
||||
|
||||
/* Commit */
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
@@ -476,8 +482,11 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
/* Update element and neighbors together */
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
ItemId itemid = PageGetItemId(page, offno);
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, itemid);
|
||||
HnswNeighborTuple ntup;
|
||||
Size etupSize;
|
||||
Size ntupSize;
|
||||
Buffer nbuf;
|
||||
Page npage;
|
||||
BlockNumber neighborPage;
|
||||
@@ -501,6 +510,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
if (ItemPointerIsValid(&etup->heaptids[0]))
|
||||
continue;
|
||||
|
||||
/* Calculate sizes */
|
||||
etupSize = ItemIdGetLength(itemid);
|
||||
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(etup->level, vacuumstate->m);
|
||||
|
||||
/* Get neighbor page */
|
||||
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
|
||||
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
|
||||
@@ -527,10 +540,13 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
for (int i = 0; i < ntup->count; i++)
|
||||
ItemPointerSetInvalid(&ntup->indextids[i]);
|
||||
|
||||
/*
|
||||
* We modified the tuples in place, no need to call
|
||||
* PageIndexTupleOverwrite
|
||||
*/
|
||||
/* Overwrite element tuple */
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Overwrite neighbor tuple */
|
||||
if (!PageIndexTupleOverwrite(npage, neighborOffno, (Item) ntup, ntupSize))
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
GenericXLogFinish(state);
|
||||
@@ -575,7 +591,6 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
|
||||
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
|
||||
vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
|
||||
vacuumstate->collation = index->rd_indcollation[0];
|
||||
vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
|
||||
vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
|
||||
"Hnsw vacuum temporary context",
|
||||
ALLOCSET_DEFAULT_SIZES);
|
||||
@@ -595,7 +610,6 @@ FreeVacuumState(HnswVacuumState * vacuumstate)
|
||||
{
|
||||
tidhash_destroy(vacuumstate->deleted);
|
||||
FreeAccessStrategy(vacuumstate->bas);
|
||||
pfree(vacuumstate->ntup);
|
||||
MemoryContextDelete(vacuumstate->tmpCtx);
|
||||
}
|
||||
|
||||
|
||||
30
src/vector.c
30
src/vector.c
@@ -866,10 +866,9 @@ vector_mul(PG_FUNCTION_ARGS)
|
||||
int
|
||||
vector_cmp_internal(Vector * a, Vector * b)
|
||||
{
|
||||
int dim = Min(a->dim, b->dim);
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Check values before dimensions to be consistent with Postgres arrays */
|
||||
for (int i = 0; i < dim; i++)
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
if (a->x[i] < b->x[i])
|
||||
return -1;
|
||||
@@ -877,13 +876,6 @@ vector_cmp_internal(Vector * a, Vector * b)
|
||||
if (a->x[i] > b->x[i])
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (a->dim < b->dim)
|
||||
return -1;
|
||||
|
||||
if (a->dim > b->dim)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -897,9 +889,6 @@ vector_lt(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
|
||||
}
|
||||
|
||||
@@ -913,9 +902,6 @@ vector_le(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
|
||||
}
|
||||
|
||||
@@ -929,9 +915,6 @@ vector_eq(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
|
||||
}
|
||||
|
||||
@@ -945,9 +928,6 @@ vector_ne(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
|
||||
}
|
||||
|
||||
@@ -961,9 +941,6 @@ vector_ge(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
|
||||
}
|
||||
|
||||
@@ -977,9 +954,6 @@ vector_gt(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -24,56 +24,6 @@ SELECT '[1e37]'::vector * '[1e37]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[1e-37]'::vector * '[1e-37]';
|
||||
ERROR: value out of range: underflow
|
||||
SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector = '[1,2]';
|
||||
ERROR: different vector dimensions 3 and 2
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
|
||||
vector_cmp
|
||||
------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2]');
|
||||
vector_cmp
|
||||
------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2]', '[2,3,4]');
|
||||
vector_cmp
|
||||
------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[2,3]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_dims('[1,2,3]');
|
||||
vector_dims
|
||||
-------------
|
||||
|
||||
@@ -6,17 +6,6 @@ SELECT '[1,2,3]'::vector * '[4,5,6]';
|
||||
SELECT '[1e37]'::vector * '[1e37]';
|
||||
SELECT '[1e-37]'::vector * '[1e-37]';
|
||||
|
||||
SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector = '[1,2]';
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
|
||||
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2]');
|
||||
SELECT vector_cmp('[1,2]', '[2,3,4]');
|
||||
SELECT vector_cmp('[2,3]', '[1,2,3]');
|
||||
|
||||
SELECT vector_dims('[1,2,3]');
|
||||
|
||||
SELECT round(vector_norm('[1,1]')::numeric, 5);
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v float4[3]);");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
|
||||
$node->safe_psql("postgres", qq(
|
||||
CREATE FUNCTION float4_l2_distance(float4[], float4[]) RETURNS float8
|
||||
AS 'BEGIN RETURN l2_distance(\$1::vector, \$2::vector); END;'
|
||||
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION float4_l2_squared_distance(float4[], float4[]) RETURNS float8
|
||||
AS 'BEGIN RETURN vector_l2_squared_distance(\$1::vector, \$2::vector); END;'
|
||||
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = float4[], RIGHTARG = float4[], PROCEDURE = float4_l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS float4_l2_ops
|
||||
FOR TYPE float4[] USING hnsw AS
|
||||
OPERATOR 1 <-> (float4[], float4[]) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 float4_l2_squared_distance(float4[], float4[]);
|
||||
));
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r1 = rand();
|
||||
my $r2 = rand();
|
||||
my $r3 = rand();
|
||||
push(@queries, "{$r1,$r2,$r3}");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->");
|
||||
my @opclasses = ("float4_l2_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Add index
|
||||
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v $opclass);");
|
||||
|
||||
my $min = $operator eq "<#>" ? 0.80 : 0.99;
|
||||
test_recall($min, $operator);
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat and hnsw access methods'
|
||||
default_version = '0.6.0'
|
||||
default_version = '0.5.1'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user