Compare commits

...

23 Commits

Author SHA1 Message Date
Andrew Kane
56cb5f3503 Version bump to 0.2.6 [skip ci] 2022-05-22 18:21:15 -07:00
Andrew Kane
ecbf46938f Moved metrics to separate function [skip ci] 2022-02-15 19:13:23 -08:00
Andrew Kane
4ca264ba02 Added Davies-Bouldin index [skip ci] 2022-02-15 19:08:47 -08:00
Andrew Kane
f5458414b8 Made recall test less flaky 2022-02-15 18:17:05 -08:00
Andrew Kane
e64ed39acb Disabled scan progress for sampling 2022-02-15 18:14:59 -08:00
Andrew Kane
0d025be9d3 Improved performance of index creation for Postgres < 12 2022-02-15 18:05:33 -08:00
Andrew Kane
fed60dce78 Added tests for lists 2022-02-15 17:04:20 -08:00
Andrew Kane
a37f5eea4a Improved compare method [skip ci] 2022-02-15 11:50:35 -08:00
Andrew Kane
4bdb27e85a Fixed style [skip ci] 2022-02-14 23:15:27 -08:00
Andrew Kane
38f869e0bd Added test for 100% recall 2022-02-14 23:13:36 -08:00
Andrew Kane
ad8acc00d4 Fixed spacing [skip ci] 2022-02-14 21:58:20 -08:00
Andrew Kane
01926a418e Print operator name for failed recall test [skip ci] 2022-02-14 21:52:31 -08:00
Andrew Kane
9658d3c1ad Made recall test less flaky 2022-02-14 21:49:08 -08:00
Andrew Kane
6b9c6516f4 Improved code 2022-02-14 21:41:27 -08:00
Andrew Kane
88be03a3fa Removed unused code 2022-02-14 21:37:53 -08:00
Andrew Kane
bf5b2c8d7e Use tuple id directly 2022-02-14 21:31:08 -08:00
Andrew Kane
ac65ec2856 Improved code [skip ci] 2022-02-14 19:38:59 -08:00
Andrew Kane
7bba0e2a01 Test recall for all operators 2022-02-14 15:22:53 -08:00
Andrew Kane
c35e9f3b84 Removed comment [skip ci] 2022-02-13 03:53:30 -08:00
Andrew Kane
9549d93260 Improved recall test 2022-02-13 03:08:39 -08:00
Andrew Kane
310809d0e5 Fixed warnings 2022-02-13 03:03:17 -08:00
Andrew Kane
ce72ca8620 Switched to heap for nearest lists for performance 2022-02-13 02:59:27 -08:00
Andrew Kane
4f2c937a1f Improved recall test 2022-02-13 02:35:28 -08:00
12 changed files with 250 additions and 83 deletions

View File

@@ -1,3 +1,7 @@
## 0.2.6 (2022-05-22)
- Improved performance of index creation for Postgres < 12
## 0.2.5 (2022-02-11) ## 0.2.5 (2022-02-11)
- Reduced memory usage during index creation - Reduced memory usage during index creation

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.2.5", "version": "0.2.6",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.2.5", "version": "0.2.6",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.2.5 EXTVERSION = 0.2.6
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*.sql) DATA = $(wildcard sql/*--*.sql)

View File

@@ -17,7 +17,7 @@ Supports L2 distance, inner product, and cosine distance
Compile and install the extension (supports Postgres 9.6+) Compile and install the extension (supports Postgres 9.6+)
```sh ```sh
git clone --branch v0.2.5 https://github.com/pgvector/pgvector.git git clone --branch v0.2.6 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -220,7 +220,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
You can also build the image manually You can also build the image manually
```sh ```sh
git clone --branch v0.2.5 https://github.com/pgvector/pgvector.git git clone --branch v0.2.6 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build -t pgvector . docker build -t pgvector .
``` ```

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.2.6'" to load this file. \quit

View File

@@ -112,13 +112,13 @@ SampleRows(IvfflatBuildState * buildstate)
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo, table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, true, targblock, 1, SampleCallback, (void *) buildstate, NULL); false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
#elif PG_VERSION_NUM >= 110000 #elif PG_VERSION_NUM >= 110000
IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo, IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
true, true, targblock, 1, SampleCallback, (void *) buildstate, NULL); false, true, targblock, 1, SampleCallback, (void *) buildstate, NULL);
#else #else
IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo, IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
true, true, targblock, 1, SampleCallback, (void *) buildstate); false, true, targblock, 1, SampleCallback, (void *) buildstate);
#endif #endif
} }
} }
@@ -167,18 +167,18 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
buildstate->inertia += minDistance; buildstate->inertia += minDistance;
buildstate->listSums[closestCenter] += minDistance;
buildstate->listCounts[closestCenter]++;
#endif #endif
/* Create a virtual tuple */ /* Create a virtual tuple */
ExecClearTuple(slot); ExecClearTuple(slot);
slot->tts_values[0] = Int32GetDatum(closestCenter); slot->tts_values[0] = Int32GetDatum(closestCenter);
slot->tts_isnull[0] = false; slot->tts_isnull[0] = false;
slot->tts_values[1] = Int32GetDatum(ItemPointerGetBlockNumberNoCheck(tid)); slot->tts_values[1] = PointerGetDatum(tid);
slot->tts_isnull[1] = false; slot->tts_isnull[1] = false;
slot->tts_values[2] = Int32GetDatum(ItemPointerGetOffsetNumberNoCheck(tid)); slot->tts_values[2] = value;
slot->tts_isnull[2] = false; slot->tts_isnull[2] = false;
slot->tts_values[3] = value;
slot->tts_isnull[3] = false;
ExecStoreVirtualTuple(slot); ExecStoreVirtualTuple(slot);
/* /*
@@ -200,8 +200,6 @@ GetNextTuple(Tuplesortstate *sortstate, TupleDesc tupdesc, TupleTableSlot *slot,
{ {
Datum value; Datum value;
bool isnull; bool isnull;
int tupblk;
int tupoff;
#if PG_VERSION_NUM >= 100000 #if PG_VERSION_NUM >= 100000
if (tuplesort_gettupleslot(sortstate, true, false, slot, NULL)) if (tuplesort_gettupleslot(sortstate, true, false, slot, NULL))
@@ -210,13 +208,11 @@ GetNextTuple(Tuplesortstate *sortstate, TupleDesc tupdesc, TupleTableSlot *slot,
#endif #endif
{ {
*list = DatumGetInt32(slot_getattr(slot, 1, &isnull)); *list = DatumGetInt32(slot_getattr(slot, 1, &isnull));
tupblk = DatumGetInt32(slot_getattr(slot, 2, &isnull)); value = slot_getattr(slot, 3, &isnull);
tupoff = DatumGetInt32(slot_getattr(slot, 3, &isnull));
value = slot_getattr(slot, 4, &isnull);
/* Form the index tuple */ /* Form the index tuple */
*itup = index_form_tuple(tupdesc, &value, &isnull); *itup = index_form_tuple(tupdesc, &value, &isnull);
ItemPointerSet(&(*itup)->t_tid, tupblk, tupoff); (*itup)->t_tid = *((ItemPointer) DatumGetPointer(slot_getattr(slot, 2, &isnull)));
} }
else else
*list = -1; *list = -1;
@@ -325,17 +321,16 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
/* Create tuple description for sorting */ /* Create tuple description for sorting */
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
buildstate->tupdesc = CreateTemplateTupleDesc(4); buildstate->tupdesc = CreateTemplateTupleDesc(3);
#else #else
buildstate->tupdesc = CreateTemplateTupleDesc(4, false); buildstate->tupdesc = CreateTemplateTupleDesc(3, false);
#endif #endif
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 1, "list", INT4OID, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 2, "blkno", INT4OID, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "offset", INT4OID, -1, 0);
#if PG_VERSION_NUM >= 110000 #if PG_VERSION_NUM >= 110000
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 4, "vector", RelationGetDescr(index)->attrs[0].atttypid, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "vector", RelationGetDescr(index)->attrs[0].atttypid, -1, 0);
#else #else
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 4, "vector", RelationGetDescr(index)->attrs[0]->atttypid, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "vector", RelationGetDescr(index)->attrs[0]->atttypid, -1, 0);
#endif #endif
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
@@ -352,6 +347,8 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
buildstate->inertia = 0; buildstate->inertia = 0;
buildstate->listSums = palloc0(sizeof(double) * buildstate->lists);
buildstate->listCounts = palloc0(sizeof(int) * buildstate->lists);
#endif #endif
} }
@@ -364,6 +361,11 @@ FreeBuildState(IvfflatBuildState * buildstate)
pfree(buildstate->centers); pfree(buildstate->centers);
pfree(buildstate->listInfo); pfree(buildstate->listInfo);
pfree(buildstate->normvec); pfree(buildstate->normvec);
#ifdef IVFFLAT_KMEANS_DEBUG
pfree(buildstate->listSums);
pfree(buildstate->listCounts);
#endif
} }
/* /*
@@ -471,6 +473,51 @@ CreateListPages(Relation index, VectorArray centers, int dimensions,
pfree(list); pfree(list);
} }
/*
* Print k-means metrics
*/
#ifdef IVFFLAT_KMEANS_DEBUG
static void
PrintKmeansMetrics(IvfflatBuildState * buildstate)
{
elog(INFO, "inertia: %.3e", buildstate->inertia);
/* Calculate Davies-Bouldin index */
if (buildstate->lists > 1)
{
double db = 0.0;
/* Calculate average distance */
for (int i = 0; i < buildstate->lists; i++)
{
if (buildstate->listCounts[i] > 0)
buildstate->listSums[i] /= buildstate->listCounts[i];
}
for (int i = 0; i < buildstate->lists; i++)
{
double max = 0.0;
double distance;
for (int j = 0; j < buildstate->lists; j++)
{
if (j == i)
continue;
distance = DatumGetFloat8(FunctionCall2Coll(buildstate->procinfo, buildstate->collation, PointerGetDatum(VectorArrayGet(buildstate->centers, i)), PointerGetDatum(VectorArrayGet(buildstate->centers, j))));
distance = (buildstate->listSums[i] + buildstate->listSums[j]) / distance;
if (distance > max)
max = distance;
}
db += max;
}
db /= buildstate->lists;
elog(INFO, "davies-bouldin: %.3f", db);
}
}
#endif
/* /*
* Create entry pages * Create entry pages
*/ */
@@ -509,7 +556,7 @@ CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
tuplesort_performsort(buildstate->sortstate); tuplesort_performsort(buildstate->sortstate);
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
elog(INFO, "inertia: %.3e", buildstate->inertia); PrintKmeansMetrics(buildstate);
#endif #endif
/* Insert */ /* Insert */

View File

@@ -62,11 +62,6 @@
#define IvfflatBench(name, code) (code) #define IvfflatBench(name, code) (code)
#endif #endif
#if PG_VERSION_NUM < 100000
#define ItemPointerGetBlockNumberNoCheck ItemPointerGetBlockNumber
#define ItemPointerGetOffsetNumberNoCheck ItemPointerGetOffsetNumber
#endif
/* Variables */ /* Variables */
extern int ivfflat_probes; extern int ivfflat_probes;
@@ -122,6 +117,8 @@ typedef struct IvfflatBuildState
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
double inertia; double inertia;
double *listSums;
int *listCounts;
#endif #endif
/* Sampling */ /* Sampling */
@@ -165,6 +162,7 @@ typedef IvfflatListData * IvfflatList;
typedef struct IvfflatScanList typedef struct IvfflatScanList
{ {
pairingheap_node ph_node;
BlockNumber startPage; BlockNumber startPage;
double distance; double distance;
} IvfflatScanList; } IvfflatScanList;
@@ -186,6 +184,8 @@ typedef struct IvfflatScanOpaqueData
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
Oid collation; Oid collation;
/* Lists */
pairingheap *listQueue;
IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */ IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */
} IvfflatScanOpaqueData; } IvfflatScanOpaqueData;

View File

@@ -1,5 +1,7 @@
#include "postgres.h" #include "postgres.h"
#include <float.h>
#include "access/relscan.h" #include "access/relscan.h"
#include "ivfflat.h" #include "ivfflat.h"
#include "miscadmin.h" #include "miscadmin.h"
@@ -17,14 +19,12 @@
* Compare list distances * Compare list distances
*/ */
static int static int
CompareLists(const void *a, const void *b) CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{ {
double diff = (((IvfflatScanList *) a)->distance - ((IvfflatScanList *) b)->distance); if (((const IvfflatScanList *) a)->distance > ((const IvfflatScanList *) b)->distance)
if (diff > 0)
return 1; return 1;
if (diff < 0) if (((const IvfflatScanList *) a)->distance < ((const IvfflatScanList *) b)->distance)
return -1; return -1;
return 0; return 0;
@@ -45,6 +45,8 @@ GetScanLists(IndexScanDesc scan, Datum value)
int listCount = 0; int listCount = 0;
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque; IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
double distance; double distance;
IvfflatScanList *scanlist;
double maxDistance = DBL_MAX;
/* Search all list pages */ /* Search all list pages */
while (BlockNumberIsValid(nextblkno)) while (BlockNumberIsValid(nextblkno))
@@ -62,22 +64,39 @@ GetScanLists(IndexScanDesc scan, Datum value)
/* Use procinfo from the index instead of scan key for performance */ /* Use procinfo from the index instead of scan key for performance */
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value)); distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
so->lists[listCount].startPage = list->startPage; if (listCount < so->probes)
so->lists[listCount].distance = distance; {
scanlist = &so->lists[listCount];
scanlist->startPage = list->startPage;
scanlist->distance = distance;
listCount++; listCount++;
/* Add to heap */
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Calculate max distance */
if (listCount == so->probes)
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
else if (distance < maxDistance)
{
/* Remove */
scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
/* Reuse */
scanlist->startPage = list->startPage;
scanlist->distance = distance;
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Update max distance */
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
} }
nextblkno = IvfflatPageGetOpaque(cpage)->nextblkno; nextblkno = IvfflatPageGetOpaque(cpage)->nextblkno;
UnlockReleaseBuffer(cbuf); UnlockReleaseBuffer(cbuf);
} }
/* Sort by distance */
/* TODO Use heap for performance */
qsort(so->lists, listCount, sizeof(IvfflatScanList), CompareLists);
if (so->probes > listCount)
so->probes = listCount;
} }
/* /*
@@ -95,7 +114,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
OffsetNumber maxoffno; OffsetNumber maxoffno;
Datum datum; Datum datum;
bool isnull; bool isnull;
int i;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation); TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
@@ -112,9 +130,9 @@ GetScanItems(IndexScanDesc scan, Datum value)
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD); BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
/* Search closest probes lists */ /* Search closest probes lists */
for (i = 0; i < so->probes; i++) while (!pairingheap_is_empty(so->listQueue))
{ {
searchPage = so->lists[i].startPage; searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
/* Search all entry pages for list */ /* Search all entry pages for list */
while (BlockNumberIsValid(searchPage)) while (BlockNumberIsValid(searchPage))
@@ -138,12 +156,10 @@ GetScanItems(IndexScanDesc scan, Datum value)
ExecClearTuple(slot); ExecClearTuple(slot);
slot->tts_values[0] = FunctionCall2Coll(so->procinfo, so->collation, datum, value); slot->tts_values[0] = FunctionCall2Coll(so->procinfo, so->collation, datum, value);
slot->tts_isnull[0] = false; slot->tts_isnull[0] = false;
slot->tts_values[1] = Int32GetDatum((int) ItemPointerGetBlockNumberNoCheck(&itup->t_tid)); slot->tts_values[1] = PointerGetDatum(&itup->t_tid);
slot->tts_isnull[1] = false; slot->tts_isnull[1] = false;
slot->tts_values[2] = Int32GetDatum((int) ItemPointerGetOffsetNumberNoCheck(&itup->t_tid)); slot->tts_values[2] = Int32GetDatum((int) searchPage);
slot->tts_isnull[2] = false; slot->tts_isnull[2] = false;
slot->tts_values[3] = Int32GetDatum((int) searchPage);
slot->tts_isnull[3] = false;
ExecStoreVirtualTuple(slot); ExecStoreVirtualTuple(slot);
tuplesort_puttupleslot(so->sortstate, slot); tuplesort_puttupleslot(so->sortstate, slot);
@@ -171,13 +187,18 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
Oid sortOperators[] = {Float8LessOperator}; Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid}; Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false}; bool nullsFirstFlags[] = {false};
int probes = ivfflat_probes;
scan = RelationGetIndexScan(index, nkeys, norderbys); scan = RelationGetIndexScan(index, nkeys, norderbys);
lists = IvfflatGetLists(scan->indexRelation); lists = IvfflatGetLists(scan->indexRelation);
so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + lists * sizeof(IvfflatScanList)); if (probes > lists)
probes = lists;
so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + probes * sizeof(IvfflatScanList));
so->buf = InvalidBuffer; so->buf = InvalidBuffer;
so->first = true; so->first = true;
so->probes = probes;
/* Set support functions */ /* Set support functions */
so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC); so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
@@ -186,14 +207,13 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
/* Create tuple description for sorting */ /* Create tuple description for sorting */
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
so->tupdesc = CreateTemplateTupleDesc(4); so->tupdesc = CreateTemplateTupleDesc(3);
#else #else
so->tupdesc = CreateTemplateTupleDesc(4, false); so->tupdesc = CreateTemplateTupleDesc(3, false);
#endif #endif
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "blkno", INT4OID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 3, "offset", INT4OID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 3, "indexblkno", INT4OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 4, "indexblkno", INT4OID, -1, 0);
/* Prep sort */ /* Prep sort */
#if PG_VERSION_NUM >= 110000 #if PG_VERSION_NUM >= 110000
@@ -208,6 +228,8 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
so->slot = MakeSingleTupleTableSlot(so->tupdesc); so->slot = MakeSingleTupleTableSlot(so->tupdesc);
#endif #endif
so->listQueue = pairingheap_allocate(CompareLists, scan);
scan->opaque = so; scan->opaque = so;
return scan; return scan;
@@ -227,7 +249,7 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
#endif #endif
so->first = true; so->first = true;
so->probes = ivfflat_probes; pairingheap_reset(so->listQueue);
if (keys && scan->numberOfKeys > 0) if (keys && scan->numberOfKeys > 0)
memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData)); memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
@@ -286,14 +308,13 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
if (tuplesort_gettupleslot(so->sortstate, true, so->slot, NULL)) if (tuplesort_gettupleslot(so->sortstate, true, so->slot, NULL))
#endif #endif
{ {
BlockNumber blkno = DatumGetInt32(slot_getattr(so->slot, 2, &so->isnull)); ItemPointer tid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
OffsetNumber offset = DatumGetInt32(slot_getattr(so->slot, 3, &so->isnull)); BlockNumber indexblkno = DatumGetInt32(slot_getattr(so->slot, 3, &so->isnull));
BlockNumber indexblkno = DatumGetInt32(slot_getattr(so->slot, 4, &so->isnull));
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
ItemPointerSet(&scan->xs_heaptid, blkno, offset); scan->xs_heaptid = *tid;
#else #else
ItemPointerSet(&scan->xs_ctup.t_self, blkno, offset); scan->xs_ctup.t_self = *tid;
#endif #endif
if (BufferIsValid(so->buf)) if (BufferIsValid(so->buf))
@@ -326,6 +347,7 @@ ivfflatendscan(IndexScanDesc scan)
if (BufferIsValid(so->buf)) if (BufferIsValid(so->buf))
ReleaseBuffer(so->buf); ReleaseBuffer(so->buf);
pairingheap_free(so->listQueue);
tuplesort_end(so->sortstate); tuplesort_end(so->sortstate);
pfree(so); pfree(so);

View File

@@ -2,15 +2,16 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More tests => 2; use Test::More tests => 9;
my $node; my $node;
my @queries = (); my @queries = ();
my @expected = (); my @expected;
my $limit = 20;
sub test_recall sub test_recall
{ {
my ($probes, $min) = @_; my ($probes, $min, $operator) = @_;
my $correct = 0; my $correct = 0;
my $total = 0; my $total = 0;
@@ -18,7 +19,7 @@ sub test_recall
my $actual = $node->safe_psql("postgres", qq( my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off; SET enable_seqscan = off;
SET ivfflat.probes = $probes; SET ivfflat.probes = $probes;
SELECT i FROM tst ORDER BY v <-> '$queries[$i]' LIMIT 10; SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
)); ));
my @actual_ids = split("\n", $actual); my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids; my %actual_set = map { $_ => 1 } @actual_ids;
@@ -33,7 +34,7 @@ sub test_recall
} }
} }
cmp_ok($correct / $total, ">=", $min); cmp_ok($correct / $total, ">=", $min, $operator);
} }
# Initialize node # Initialize node
@@ -56,17 +57,32 @@ for (1..20) {
push(@queries, "[$r1,$r2,$r3]"); push(@queries, "[$r1,$r2,$r3]");
} }
# Check each index type
my @operators = ("<->", "<#>", "<=>");
foreach (@operators) {
my $operator = $_;
# Get exact results # Get exact results
@expected = ();
foreach (@queries) { foreach (@queries) {
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v <-> '$_' LIMIT 10;"); my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
push(@expected, $res); push(@expected, $res);
} }
# Add index # Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);"); my $opclass;
if ($operator == "<->") {
$opclass = "vector_l2_ops";
} elsif ($operator == "<#>") {
$opclass = "vector_ip_ops";
} else {
$opclass = "vector_cosine_ops";
}
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test approximate results # Test approximate results
test_recall(1, 0.8); test_recall(1, 0.75, $operator);
test_recall(10, 0.95, $operator);
# Test probes test_recall(100, 1.0, $operator);
test_recall(100, 1.0); }

View File

@@ -0,0 +1,45 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 60;
# Initialize node
my $node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4 primary key, v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
# Check each index type
my @operators = ("<->", "<#>", "<=>");
foreach (@operators) {
my $operator = $_;
# Add index
my $opclass;
if ($operator == "<->") {
$opclass = "vector_l2_ops";
} elsif ($operator == "<#>") {
$opclass = "vector_ip_ops";
} else {
$opclass = "vector_cosine_ops";
}
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test 100% recall
for (1..20) {
my $i = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $i;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
}

31
test/t/006_lists.pl Normal file
View File

@@ -0,0 +1,31 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 3;
# Initialize node
my $node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX lists50 ON tst USING ivfflat (v) WITH (lists = 50);");
$node->safe_psql("postgres", "CREATE INDEX lists100 ON tst USING ivfflat (v) WITH (lists = 100);");
# Test prefers more lists
my $res = $node->safe_psql("postgres", "EXPLAIN SELECT v FROM tst ORDER BY v <-> '[0.5,0.5,0.5]' LIMIT 10;");
like($res, qr/lists100/);
unlike($res, qr/lists50/);
# Test errors with too much memory
my ($ret, $stdout, $stderr) = $node->psql("postgres",
"CREATE INDEX lists10000 ON tst USING ivfflat (v) WITH (lists = 10000);"
);
like($stderr, qr/memory required is/);

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat access method' comment = 'vector data type and ivfflat access method'
default_version = '0.2.5' default_version = '0.2.6'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true