Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
7c6694e0ef Added bound option 2022-02-13 01:41:06 -08:00
13 changed files with 93 additions and 250 deletions

View File

@@ -1,7 +1,3 @@
## 0.2.6 (2022-05-22)
- Improved performance of index creation for Postgres < 12
## 0.2.5 (2022-02-11) ## 0.2.5 (2022-02-11)
- Reduced memory usage during index creation - Reduced memory usage during index creation

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.2.6", "version": "0.2.5",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.2.6", "version": "0.2.5",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.2.6 EXTVERSION = 0.2.5
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*.sql) DATA = $(wildcard sql/*--*.sql)

View File

@@ -17,7 +17,7 @@ Supports L2 distance, inner product, and cosine distance
Compile and install the extension (supports Postgres 9.6+) Compile and install the extension (supports Postgres 9.6+)
```sh ```sh
git clone --branch v0.2.6 https://github.com/pgvector/pgvector.git git clone --branch v0.2.5 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -220,7 +220,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
You can also build the image manually You can also build the image manually
```sh ```sh
git clone --branch v0.2.6 https://github.com/pgvector/pgvector.git git clone --branch v0.2.5 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build -t pgvector . docker build -t pgvector .
``` ```

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.2.6'" to load this file. \quit

View File

@@ -112,13 +112,13 @@ SampleRows(IvfflatBuildState * buildstate)
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo, table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL); false, true, true, targblock, 1, SampleCallback, (void *) buildstate, NULL);
#elif PG_VERSION_NUM >= 110000 #elif PG_VERSION_NUM >= 110000
IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo, IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, targblock, 1, SampleCallback, (void *) buildstate, NULL); true, true, targblock, 1, SampleCallback, (void *) buildstate, NULL);
#else #else
IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo, IndexBuildHeapRangeScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, targblock, 1, SampleCallback, (void *) buildstate); true, true, targblock, 1, SampleCallback, (void *) buildstate);
#endif #endif
} }
} }
@@ -167,18 +167,18 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
buildstate->inertia += minDistance; buildstate->inertia += minDistance;
buildstate->listSums[closestCenter] += minDistance;
buildstate->listCounts[closestCenter]++;
#endif #endif
/* Create a virtual tuple */ /* Create a virtual tuple */
ExecClearTuple(slot); ExecClearTuple(slot);
slot->tts_values[0] = Int32GetDatum(closestCenter); slot->tts_values[0] = Int32GetDatum(closestCenter);
slot->tts_isnull[0] = false; slot->tts_isnull[0] = false;
slot->tts_values[1] = PointerGetDatum(tid); slot->tts_values[1] = Int32GetDatum(ItemPointerGetBlockNumberNoCheck(tid));
slot->tts_isnull[1] = false; slot->tts_isnull[1] = false;
slot->tts_values[2] = value; slot->tts_values[2] = Int32GetDatum(ItemPointerGetOffsetNumberNoCheck(tid));
slot->tts_isnull[2] = false; slot->tts_isnull[2] = false;
slot->tts_values[3] = value;
slot->tts_isnull[3] = false;
ExecStoreVirtualTuple(slot); ExecStoreVirtualTuple(slot);
/* /*
@@ -200,6 +200,8 @@ GetNextTuple(Tuplesortstate *sortstate, TupleDesc tupdesc, TupleTableSlot *slot,
{ {
Datum value; Datum value;
bool isnull; bool isnull;
int tupblk;
int tupoff;
#if PG_VERSION_NUM >= 100000 #if PG_VERSION_NUM >= 100000
if (tuplesort_gettupleslot(sortstate, true, false, slot, NULL)) if (tuplesort_gettupleslot(sortstate, true, false, slot, NULL))
@@ -208,11 +210,13 @@ GetNextTuple(Tuplesortstate *sortstate, TupleDesc tupdesc, TupleTableSlot *slot,
#endif #endif
{ {
*list = DatumGetInt32(slot_getattr(slot, 1, &isnull)); *list = DatumGetInt32(slot_getattr(slot, 1, &isnull));
value = slot_getattr(slot, 3, &isnull); tupblk = DatumGetInt32(slot_getattr(slot, 2, &isnull));
tupoff = DatumGetInt32(slot_getattr(slot, 3, &isnull));
value = slot_getattr(slot, 4, &isnull);
/* Form the index tuple */ /* Form the index tuple */
*itup = index_form_tuple(tupdesc, &value, &isnull); *itup = index_form_tuple(tupdesc, &value, &isnull);
(*itup)->t_tid = *((ItemPointer) DatumGetPointer(slot_getattr(slot, 2, &isnull))); ItemPointerSet(&(*itup)->t_tid, tupblk, tupoff);
} }
else else
*list = -1; *list = -1;
@@ -321,16 +325,17 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
/* Create tuple description for sorting */ /* Create tuple description for sorting */
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
buildstate->tupdesc = CreateTemplateTupleDesc(3); buildstate->tupdesc = CreateTemplateTupleDesc(4);
#else #else
buildstate->tupdesc = CreateTemplateTupleDesc(3, false); buildstate->tupdesc = CreateTemplateTupleDesc(4, false);
#endif #endif
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 1, "list", INT4OID, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 2, "blkno", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "offset", INT4OID, -1, 0);
#if PG_VERSION_NUM >= 110000 #if PG_VERSION_NUM >= 110000
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "vector", RelationGetDescr(index)->attrs[0].atttypid, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 4, "vector", RelationGetDescr(index)->attrs[0].atttypid, -1, 0);
#else #else
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "vector", RelationGetDescr(index)->attrs[0]->atttypid, -1, 0); TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 4, "vector", RelationGetDescr(index)->attrs[0]->atttypid, -1, 0);
#endif #endif
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
@@ -347,8 +352,6 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
buildstate->inertia = 0; buildstate->inertia = 0;
buildstate->listSums = palloc0(sizeof(double) * buildstate->lists);
buildstate->listCounts = palloc0(sizeof(int) * buildstate->lists);
#endif #endif
} }
@@ -361,11 +364,6 @@ FreeBuildState(IvfflatBuildState * buildstate)
pfree(buildstate->centers); pfree(buildstate->centers);
pfree(buildstate->listInfo); pfree(buildstate->listInfo);
pfree(buildstate->normvec); pfree(buildstate->normvec);
#ifdef IVFFLAT_KMEANS_DEBUG
pfree(buildstate->listSums);
pfree(buildstate->listCounts);
#endif
} }
/* /*
@@ -473,51 +471,6 @@ CreateListPages(Relation index, VectorArray centers, int dimensions,
pfree(list); pfree(list);
} }
/*
* Print k-means metrics
*/
#ifdef IVFFLAT_KMEANS_DEBUG
static void
PrintKmeansMetrics(IvfflatBuildState * buildstate)
{
elog(INFO, "inertia: %.3e", buildstate->inertia);
/* Calculate Davies-Bouldin index */
if (buildstate->lists > 1)
{
double db = 0.0;
/* Calculate average distance */
for (int i = 0; i < buildstate->lists; i++)
{
if (buildstate->listCounts[i] > 0)
buildstate->listSums[i] /= buildstate->listCounts[i];
}
for (int i = 0; i < buildstate->lists; i++)
{
double max = 0.0;
double distance;
for (int j = 0; j < buildstate->lists; j++)
{
if (j == i)
continue;
distance = DatumGetFloat8(FunctionCall2Coll(buildstate->procinfo, buildstate->collation, PointerGetDatum(VectorArrayGet(buildstate->centers, i)), PointerGetDatum(VectorArrayGet(buildstate->centers, j))));
distance = (buildstate->listSums[i] + buildstate->listSums[j]) / distance;
if (distance > max)
max = distance;
}
db += max;
}
db /= buildstate->lists;
elog(INFO, "davies-bouldin: %.3f", db);
}
}
#endif
/* /*
* Create entry pages * Create entry pages
*/ */
@@ -556,7 +509,7 @@ CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
tuplesort_performsort(buildstate->sortstate); tuplesort_performsort(buildstate->sortstate);
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
PrintKmeansMetrics(buildstate); elog(INFO, "inertia: %.3e", buildstate->inertia);
#endif #endif
/* Insert */ /* Insert */

View File

@@ -13,6 +13,7 @@
#endif #endif
int ivfflat_probes; int ivfflat_probes;
int ivfflat_bound;
static relopt_kind ivfflat_relopt_kind; static relopt_kind ivfflat_relopt_kind;
/* /*
@@ -32,6 +33,10 @@ _PG_init(void)
DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes", DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
"Valid range is 1..lists.", &ivfflat_probes, "Valid range is 1..lists.", &ivfflat_probes,
1, 1, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL); 1, 1, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
DefineCustomIntVariable("ivfflat.bound", "Sets the max results from index (experimental)",
NULL, &ivfflat_bound,
0, 0, INT_MAX, PGC_USERSET, 0, NULL, NULL, NULL);
} }
/* /*

View File

@@ -62,8 +62,14 @@
#define IvfflatBench(name, code) (code) #define IvfflatBench(name, code) (code)
#endif #endif
#if PG_VERSION_NUM < 100000
#define ItemPointerGetBlockNumberNoCheck ItemPointerGetBlockNumber
#define ItemPointerGetOffsetNumberNoCheck ItemPointerGetOffsetNumber
#endif
/* Variables */ /* Variables */
extern int ivfflat_probes; extern int ivfflat_probes;
extern int ivfflat_bound;
typedef struct VectorArrayData typedef struct VectorArrayData
{ {
@@ -117,8 +123,6 @@ typedef struct IvfflatBuildState
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
double inertia; double inertia;
double *listSums;
int *listCounts;
#endif #endif
/* Sampling */ /* Sampling */
@@ -162,7 +166,6 @@ typedef IvfflatListData * IvfflatList;
typedef struct IvfflatScanList typedef struct IvfflatScanList
{ {
pairingheap_node ph_node;
BlockNumber startPage; BlockNumber startPage;
double distance; double distance;
} IvfflatScanList; } IvfflatScanList;
@@ -184,8 +187,6 @@ typedef struct IvfflatScanOpaqueData
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
Oid collation; Oid collation;
/* Lists */
pairingheap *listQueue;
IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */ IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */
} IvfflatScanOpaqueData; } IvfflatScanOpaqueData;

View File

@@ -1,7 +1,5 @@
#include "postgres.h" #include "postgres.h"
#include <float.h>
#include "access/relscan.h" #include "access/relscan.h"
#include "ivfflat.h" #include "ivfflat.h"
#include "miscadmin.h" #include "miscadmin.h"
@@ -19,12 +17,14 @@
* Compare list distances * Compare list distances
*/ */
static int static int
CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg) CompareLists(const void *a, const void *b)
{ {
if (((const IvfflatScanList *) a)->distance > ((const IvfflatScanList *) b)->distance) double diff = (((IvfflatScanList *) a)->distance - ((IvfflatScanList *) b)->distance);
if (diff > 0)
return 1; return 1;
if (((const IvfflatScanList *) a)->distance < ((const IvfflatScanList *) b)->distance) if (diff < 0)
return -1; return -1;
return 0; return 0;
@@ -45,8 +45,6 @@ GetScanLists(IndexScanDesc scan, Datum value)
int listCount = 0; int listCount = 0;
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque; IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
double distance; double distance;
IvfflatScanList *scanlist;
double maxDistance = DBL_MAX;
/* Search all list pages */ /* Search all list pages */
while (BlockNumberIsValid(nextblkno)) while (BlockNumberIsValid(nextblkno))
@@ -64,39 +62,22 @@ GetScanLists(IndexScanDesc scan, Datum value)
/* Use procinfo from the index instead of scan key for performance */ /* Use procinfo from the index instead of scan key for performance */
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value)); distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
if (listCount < so->probes) so->lists[listCount].startPage = list->startPage;
{ so->lists[listCount].distance = distance;
scanlist = &so->lists[listCount]; listCount++;
scanlist->startPage = list->startPage;
scanlist->distance = distance;
listCount++;
/* Add to heap */
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Calculate max distance */
if (listCount == so->probes)
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
else if (distance < maxDistance)
{
/* Remove */
scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
/* Reuse */
scanlist->startPage = list->startPage;
scanlist->distance = distance;
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Update max distance */
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
} }
nextblkno = IvfflatPageGetOpaque(cpage)->nextblkno; nextblkno = IvfflatPageGetOpaque(cpage)->nextblkno;
UnlockReleaseBuffer(cbuf); UnlockReleaseBuffer(cbuf);
} }
/* Sort by distance */
/* TODO Use heap for performance */
qsort(so->lists, listCount, sizeof(IvfflatScanList), CompareLists);
if (so->probes > listCount)
so->probes = listCount;
} }
/* /*
@@ -114,6 +95,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
OffsetNumber maxoffno; OffsetNumber maxoffno;
Datum datum; Datum datum;
bool isnull; bool isnull;
int i;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation); TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
@@ -129,10 +111,14 @@ GetScanItems(IndexScanDesc scan, Datum value)
*/ */
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD); BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
/* Set the max number of results */
if (ivfflat_bound > 0)
tuplesort_set_bound(so->sortstate, ivfflat_bound);
/* Search closest probes lists */ /* Search closest probes lists */
while (!pairingheap_is_empty(so->listQueue)) for (i = 0; i < so->probes; i++)
{ {
searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage; searchPage = so->lists[i].startPage;
/* Search all entry pages for list */ /* Search all entry pages for list */
while (BlockNumberIsValid(searchPage)) while (BlockNumberIsValid(searchPage))
@@ -156,10 +142,12 @@ GetScanItems(IndexScanDesc scan, Datum value)
ExecClearTuple(slot); ExecClearTuple(slot);
slot->tts_values[0] = FunctionCall2Coll(so->procinfo, so->collation, datum, value); slot->tts_values[0] = FunctionCall2Coll(so->procinfo, so->collation, datum, value);
slot->tts_isnull[0] = false; slot->tts_isnull[0] = false;
slot->tts_values[1] = PointerGetDatum(&itup->t_tid); slot->tts_values[1] = Int32GetDatum((int) ItemPointerGetBlockNumberNoCheck(&itup->t_tid));
slot->tts_isnull[1] = false; slot->tts_isnull[1] = false;
slot->tts_values[2] = Int32GetDatum((int) searchPage); slot->tts_values[2] = Int32GetDatum((int) ItemPointerGetOffsetNumberNoCheck(&itup->t_tid));
slot->tts_isnull[2] = false; slot->tts_isnull[2] = false;
slot->tts_values[3] = Int32GetDatum((int) searchPage);
slot->tts_isnull[3] = false;
ExecStoreVirtualTuple(slot); ExecStoreVirtualTuple(slot);
tuplesort_puttupleslot(so->sortstate, slot); tuplesort_puttupleslot(so->sortstate, slot);
@@ -187,18 +175,13 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
Oid sortOperators[] = {Float8LessOperator}; Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid}; Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false}; bool nullsFirstFlags[] = {false};
int probes = ivfflat_probes;
scan = RelationGetIndexScan(index, nkeys, norderbys); scan = RelationGetIndexScan(index, nkeys, norderbys);
lists = IvfflatGetLists(scan->indexRelation); lists = IvfflatGetLists(scan->indexRelation);
if (probes > lists) so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + lists * sizeof(IvfflatScanList));
probes = lists;
so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + probes * sizeof(IvfflatScanList));
so->buf = InvalidBuffer; so->buf = InvalidBuffer;
so->first = true; so->first = true;
so->probes = probes;
/* Set support functions */ /* Set support functions */
so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC); so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
@@ -207,13 +190,14 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
/* Create tuple description for sorting */ /* Create tuple description for sorting */
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
so->tupdesc = CreateTemplateTupleDesc(3); so->tupdesc = CreateTemplateTupleDesc(4);
#else #else
so->tupdesc = CreateTemplateTupleDesc(3, false); so->tupdesc = CreateTemplateTupleDesc(4, false);
#endif #endif
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "blkno", INT4OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 3, "indexblkno", INT4OID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 3, "offset", INT4OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 4, "indexblkno", INT4OID, -1, 0);
/* Prep sort */ /* Prep sort */
#if PG_VERSION_NUM >= 110000 #if PG_VERSION_NUM >= 110000
@@ -228,8 +212,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
so->slot = MakeSingleTupleTableSlot(so->tupdesc); so->slot = MakeSingleTupleTableSlot(so->tupdesc);
#endif #endif
so->listQueue = pairingheap_allocate(CompareLists, scan);
scan->opaque = so; scan->opaque = so;
return scan; return scan;
@@ -249,7 +231,7 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
#endif #endif
so->first = true; so->first = true;
pairingheap_reset(so->listQueue); so->probes = ivfflat_probes;
if (keys && scan->numberOfKeys > 0) if (keys && scan->numberOfKeys > 0)
memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData)); memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
@@ -308,13 +290,14 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
if (tuplesort_gettupleslot(so->sortstate, true, so->slot, NULL)) if (tuplesort_gettupleslot(so->sortstate, true, so->slot, NULL))
#endif #endif
{ {
ItemPointer tid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull)); BlockNumber blkno = DatumGetInt32(slot_getattr(so->slot, 2, &so->isnull));
BlockNumber indexblkno = DatumGetInt32(slot_getattr(so->slot, 3, &so->isnull)); OffsetNumber offset = DatumGetInt32(slot_getattr(so->slot, 3, &so->isnull));
BlockNumber indexblkno = DatumGetInt32(slot_getattr(so->slot, 4, &so->isnull));
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
scan->xs_heaptid = *tid; ItemPointerSet(&scan->xs_heaptid, blkno, offset);
#else #else
scan->xs_ctup.t_self = *tid; ItemPointerSet(&scan->xs_ctup.t_self, blkno, offset);
#endif #endif
if (BufferIsValid(so->buf)) if (BufferIsValid(so->buf))
@@ -347,7 +330,6 @@ ivfflatendscan(IndexScanDesc scan)
if (BufferIsValid(so->buf)) if (BufferIsValid(so->buf))
ReleaseBuffer(so->buf); ReleaseBuffer(so->buf);
pairingheap_free(so->listQueue);
tuplesort_end(so->sortstate); tuplesort_end(so->sortstate);
pfree(so); pfree(so);

View File

@@ -2,16 +2,15 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More tests => 9; use Test::More tests => 2;
my $node; my $node;
my @queries = (); my @queries = ();
my @expected; my @expected = ();
my $limit = 20;
sub test_recall sub test_recall
{ {
my ($probes, $min, $operator) = @_; my ($probes, $min) = @_;
my $correct = 0; my $correct = 0;
my $total = 0; my $total = 0;
@@ -19,7 +18,7 @@ sub test_recall
my $actual = $node->safe_psql("postgres", qq( my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off; SET enable_seqscan = off;
SET ivfflat.probes = $probes; SET ivfflat.probes = $probes;
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit; SELECT i FROM tst ORDER BY v <-> '$queries[$i]' LIMIT 10;
)); ));
my @actual_ids = split("\n", $actual); my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids; my %actual_set = map { $_ => 1 } @actual_ids;
@@ -34,7 +33,7 @@ sub test_recall
} }
} }
cmp_ok($correct / $total, ">=", $min, $operator); cmp_ok($correct / $total, ">=", $min);
} }
# Initialize node # Initialize node
@@ -57,32 +56,17 @@ for (1..20) {
push(@queries, "[$r1,$r2,$r3]"); push(@queries, "[$r1,$r2,$r3]");
} }
# Check each index type # Get exact results
my @operators = ("<->", "<#>", "<=>"); foreach (@queries) {
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v <-> '$_' LIMIT 10;");
foreach (@operators) { push(@expected, $res);
my $operator = $_;
# Get exact results
@expected = ();
foreach (@queries) {
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
push(@expected, $res);
}
# Add index
my $opclass;
if ($operator == "<->") {
$opclass = "vector_l2_ops";
} elsif ($operator == "<#>") {
$opclass = "vector_ip_ops";
} else {
$opclass = "vector_cosine_ops";
}
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test approximate results
test_recall(1, 0.75, $operator);
test_recall(10, 0.95, $operator);
test_recall(100, 1.0, $operator);
} }
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
# Test approximate results
test_recall(1, 0.8);
# Test probes
test_recall(100, 1.0);

View File

@@ -1,45 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 60;
# Initialize node
my $node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4 primary key, v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
# Check each index type
my @operators = ("<->", "<#>", "<=>");
foreach (@operators) {
my $operator = $_;
# Add index
my $opclass;
if ($operator == "<->") {
$opclass = "vector_l2_ops";
} elsif ($operator == "<#>") {
$opclass = "vector_ip_ops";
} else {
$opclass = "vector_cosine_ops";
}
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test 100% recall
for (1..20) {
my $i = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $i;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
}

View File

@@ -1,31 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 3;
# Initialize node
my $node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX lists50 ON tst USING ivfflat (v) WITH (lists = 50);");
$node->safe_psql("postgres", "CREATE INDEX lists100 ON tst USING ivfflat (v) WITH (lists = 100);");
# Test prefers more lists
my $res = $node->safe_psql("postgres", "EXPLAIN SELECT v FROM tst ORDER BY v <-> '[0.5,0.5,0.5]' LIMIT 10;");
like($res, qr/lists100/);
unlike($res, qr/lists50/);
# Test errors with too much memory
my ($ret, $stdout, $stderr) = $node->psql("postgres",
"CREATE INDEX lists10000 ON tst USING ivfflat (v) WITH (lists = 10000);"
);
like($stderr, qr/memory required is/);

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat access method' comment = 'vector data type and ivfflat access method'
default_version = '0.2.6' default_version = '0.2.5'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true