mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-21 19:47:35 +08:00
Compare commits
6 Commits
v0.8.5
...
ivfflat-qu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c599017441 | ||
|
|
f341cea329 | ||
|
|
73356ecfa7 | ||
|
|
0e557b1d18 | ||
|
|
769a60884c | ||
|
|
8711840058 |
10
README.md
10
README.md
@@ -465,6 +465,16 @@ If filtering by many different values, consider [partitioning](https://www.postg
|
||||
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
|
||||
```
|
||||
|
||||
## Multitenancy
|
||||
|
||||
For applications with multiple tenants, sharing an approximate index between tenants means vectors from one tenant can affect recall (and speed) for other tenants.
|
||||
|
||||
For tenant isolation, use [list partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) or separate tables.
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (customer_id int, embedding vector(3)) PARTITION BY LIST(customer_id);
|
||||
```
|
||||
|
||||
## Iterative Index Scans
|
||||
|
||||
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "fmgr.h"
|
||||
#include "ivfflat.h"
|
||||
#include "miscadmin.h"
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "utils/relcache.h"
|
||||
|
||||
@@ -105,16 +106,44 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
|
||||
}
|
||||
|
||||
/*
|
||||
* Quick approach if we have no data
|
||||
* Check if vector array contains a vector
|
||||
*/
|
||||
static bool
|
||||
VectorArrayContains(VectorArray arr, Pointer val)
|
||||
{
|
||||
Datum d = PointerGetDatum(val);
|
||||
|
||||
for (int i = 0; i < arr->length; i++)
|
||||
{
|
||||
if (datumIsEqual(d, PointerGetDatum(VectorArrayGet(arr, i)), false, -1))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Quick approach if we have little data
|
||||
*/
|
||||
static void
|
||||
RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
|
||||
QuickCenters(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
|
||||
{
|
||||
int dimensions = centers->dim;
|
||||
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
|
||||
Oid collation = index->rd_indcollation[0];
|
||||
float *x = (float *) palloc(sizeof(float) * dimensions);
|
||||
|
||||
/* Fill with unique samples (already normalized) */
|
||||
for (int i = 0; i < samples->length; i++)
|
||||
{
|
||||
Pointer sample = VectorArrayGet(samples, i);
|
||||
|
||||
if (!VectorArrayContains(centers, sample))
|
||||
{
|
||||
VectorArraySet(centers, centers->length, sample);
|
||||
centers->length++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Fill with random data */
|
||||
while (centers->length < centers->maxlen)
|
||||
{
|
||||
@@ -548,8 +577,8 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
|
||||
ALLOCSET_DEFAULT_SIZES);
|
||||
MemoryContext oldCtx = MemoryContextSwitchTo(kmeansCtx);
|
||||
|
||||
if (samples->length == 0)
|
||||
RandomCenters(index, centers, typeInfo);
|
||||
if (samples->length <= centers->maxlen)
|
||||
QuickCenters(index, samples, centers, typeInfo);
|
||||
else
|
||||
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ VectorArrayInit(int maxlen, int dimensions, Size itemsize)
|
||||
{
|
||||
VectorArray res;
|
||||
|
||||
if (maxlen < 1 || dimensions < 1)
|
||||
elog(ERROR, "safety check failed");
|
||||
if (maxlen < 1 || dimensions < 1 || itemsize == 0)
|
||||
elog(ERROR, "cannot create vector array");
|
||||
|
||||
/* Ensure items are aligned to prevent UB */
|
||||
itemsize = MAXALIGN(itemsize);
|
||||
|
||||
Reference in New Issue
Block a user