liuyu/pgvector
1
0
Fork 0
mirror of https://github.com/pgvector/pgvector.git synced 2026-07-12 07:36:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use List for samples

Browse Source
This commit is contained in:
Andrew Kane
2023-10-16 15:32:51 -07:00
parent e630efd195
commit 64223989cd
3 changed files with 72 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/ivfbuild.c
View File

@@ -11,6 +11,7 @@
#include "miscadmin.h" #include "miscadmin.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "tcop/tcopprot.h" #include "tcop/tcopprot.h"
#include "utils/datum.h"
#include "utils/memutils.h" #include "utils/memutils.h"
#if PG_VERSION_NUM >= 140000 #if PG_VERSION_NUM >= 140000
@@ -65,11 +66,18 @@
static void static void
AddSample(Datum *values, IvfflatBuildState * buildstate) AddSample(Datum *values, IvfflatBuildState * buildstate)
{ {
VectorArray samples = buildstate->samples; MemoryContext oldCtx;
int targsamples = samples->maxlen; Datum value;
int targsamples = buildstate->targsamples;
/* Use memory context since detoast can allocate */
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Detoast once for all calls */ /* Detoast once for all calls */
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Restore memory context */
MemoryContextSwitchTo(oldCtx);
/* /*
* Normalize with KMEANS_NORM_PROC since spherical distance function * Normalize with KMEANS_NORM_PROC since spherical distance function
@@ -81,18 +89,23 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
return; return;
} }
if (samples->length < targsamples) /* Copy datum */
{ value = datumCopy(value, false, -1);
VectorArraySet(samples, samples->length, DatumGetVector(value));
samples->length++; /* Reset memory context */
} MemoryContextReset(buildstate->tmpCtx);
if (list_length(buildstate->samples) < targsamples)
buildstate->samples = lappend(buildstate->samples, DatumGetVector(value));
else else
{ {
if (buildstate->rowstoskip < 0) if (buildstate->rowstoskip < 0)
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples); buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, list_length(buildstate->samples), targsamples);
if (buildstate->rowstoskip <= 0) if (buildstate->rowstoskip <= 0)
{ {
ListCell *lc;
#if PG_VERSION_NUM >= 150000 #if PG_VERSION_NUM >= 150000
int k = (int) (targsamples * sampler_random_fract(&buildstate->rstate.randstate)); int k = (int) (targsamples * sampler_random_fract(&buildstate->rstate.randstate));
#else #else
@@ -100,7 +113,8 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
#endif #endif
Assert(k >= 0 && k < targsamples); Assert(k >= 0 && k < targsamples);
VectorArraySet(samples, k, DatumGetVector(value)); lc = list_nth_cell(buildstate->samples, k);
lfirst(lc) = DatumGetVector(value);
} }
buildstate->rowstoskip -= 1; buildstate->rowstoskip -= 1;
@@ -115,21 +129,13 @@ SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state) bool *isnull, bool tupleIsAlive, void *state)
{ {
IvfflatBuildState *buildstate = (IvfflatBuildState *) state; IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
MemoryContext oldCtx;
/* Skip nulls */ /* Skip nulls */
if (isnull[0]) if (isnull[0])
return; return;
/* Use memory context since detoast can allocate */
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Add sample */ /* Add sample */
AddSample(values, state); AddSample(values, buildstate);
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
MemoryContextReset(buildstate->tmpCtx);
} }
/* /*
@@ -138,7 +144,7 @@ SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
static void static void
SampleRows(IvfflatBuildState * buildstate) SampleRows(IvfflatBuildState * buildstate)
{ {
int targsamples = buildstate->samples->maxlen; int targsamples = buildstate->targsamples;
BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap); BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
buildstate->rowstoskip = -1; buildstate->rowstoskip = -1;
@@ -449,12 +455,13 @@ ComputeCenters(IvfflatBuildState * buildstate)
/* Sample rows */ /* Sample rows */
/* TODO Ensure within maintenance_work_mem */ /* TODO Ensure within maintenance_work_mem */
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions); buildstate->samples = NIL;
buildstate->targsamples = numSamples;
if (buildstate->heap != NULL) if (buildstate->heap != NULL)
{ {
SampleRows(buildstate); SampleRows(buildstate);
if (buildstate->samples->length < buildstate->lists) if (list_length(buildstate->samples) < buildstate->lists)
{ {
ereport(NOTICE, ereport(NOTICE,
(errmsg("ivfflat index created with little data"), (errmsg("ivfflat index created with little data"),
@@ -467,7 +474,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers)); IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
/* Free samples before we allocate more memory */ /* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples); list_free_deep(buildstate->samples);
} }
/* /*

9
src/ivfflat.h
View File

@@ -80,6 +80,10 @@
#define RandomInt() random() #define RandomInt() random()
#endif #endif
#if PG_VERSION_NUM < 130000
#define list_sort(list, cmp) list_qsort(list, cmp)
#endif
/* Variables */ /* Variables */
extern int ivfflat_probes; extern int ivfflat_probes;
@@ -178,7 +182,8 @@ typedef struct IvfflatBuildState
Oid collation; Oid collation;
/* Variables */ /* Variables */
VectorArray samples; List *samples;
int targsamples;
VectorArray centers; VectorArray centers;
ListInfo *listInfo; ListInfo *listInfo;
Vector *normvec; Vector *normvec;
@@ -274,7 +279,7 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
VectorArray VectorArrayInit(int maxlen, int dimensions); VectorArray VectorArrayInit(int maxlen, int dimensions);
void VectorArrayFree(VectorArray arr); void VectorArrayFree(VectorArray arr);
void PrintVectorArray(char *msg, VectorArray arr); void PrintVectorArray(char *msg, VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers); void IvfflatKmeans(Relation index, List *samples, VectorArray centers);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum); FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result); bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
int IvfflatGetLists(Relation index); int IvfflatGetLists(Relation index);

54
src/ivfkmeans.c
View File

@@ -12,20 +12,20 @@
* https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf * https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf
*/ */
static void static void
InitCenters(Relation index, VectorArray samples, VectorArray centers, float *lowerBound) InitCenters(Relation index, List *samples, VectorArray centers, float *lowerBound)
{ {
FmgrInfo *procinfo; FmgrInfo *procinfo;
Oid collation; Oid collation;
int64 j; int64 j;
float *weight = palloc(samples->length * sizeof(float)); float *weight = palloc(list_length(samples) * sizeof(float));
int numCenters = centers->maxlen; int numCenters = centers->maxlen;
int numSamples = samples->length; int numSamples = list_length(samples);
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC); procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
collation = index->rd_indcollation[0]; collation = index->rd_indcollation[0];
/* Choose an initial center uniformly at random */ /* Choose an initial center uniformly at random */
VectorArraySet(centers, 0, VectorArrayGet(samples, RandomInt() % samples->length)); VectorArraySet(centers, 0, list_nth(samples, RandomInt() % list_length(samples)));
centers->length++; centers->length++;
for (j = 0; j < numSamples; j++) for (j = 0; j < numSamples; j++)
@@ -42,7 +42,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
for (j = 0; j < numSamples; j++) for (j = 0; j < numSamples; j++)
{ {
Vector *vec = VectorArrayGet(samples, j); Vector *vec = list_nth(samples, j);
double distance; double distance;
/* Only need to compute distance for new center */ /* Only need to compute distance for new center */
@@ -74,7 +74,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
break; break;
} }
VectorArraySet(centers, i + 1, VectorArrayGet(samples, j)); VectorArraySet(centers, i + 1, list_nth(samples, j));
centers->length++; centers->length++;
} }
@@ -106,25 +106,41 @@ CompareVectors(const void *a, const void *b)
return vector_cmp_internal((Vector *) a, (Vector *) b); return vector_cmp_internal((Vector *) a, (Vector *) b);
} }
/*
* Compare list vectors
*/
static int
#if PG_VERSION_NUM >= 130000
CompareListVectors(const ListCell *a, const ListCell *b)
#else
CompareListVectors(const void *a, const void *b)
#endif
{
Vector *va = lfirst((ListCell *) a);
Vector *vb = lfirst((ListCell *) b);
return CompareVectors(va, vb);
}
/* /*
* Quick approach if we have little data * Quick approach if we have little data
*/ */
static void static void
QuickCenters(Relation index, VectorArray samples, VectorArray centers) QuickCenters(Relation index, List *samples, VectorArray centers)
{ {
int dimensions = centers->dim; int dimensions = centers->dim;
Oid collation = index->rd_indcollation[0]; Oid collation = index->rd_indcollation[0];
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC); FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
/* Copy existing vectors while avoiding duplicates */ /* Copy existing vectors while avoiding duplicates */
if (samples->length > 0) if (list_length(samples) > 0)
{ {
qsort(samples->items, samples->length, VECTOR_SIZE(samples->dim), CompareVectors); list_sort(samples, CompareListVectors);
for (int i = 0; i < samples->length; i++) for (int i = 0; i < list_length(samples); i++)
{ {
Vector *vec = VectorArrayGet(samples, i); Vector *vec = list_nth(samples, i);
if (i == 0 || CompareVectors(vec, VectorArrayGet(samples, i - 1)) != 0) if (i == 0 || CompareVectors(vec, list_nth(samples, i - 1)) != 0)
{ {
VectorArraySet(centers, centers->length, vec); VectorArraySet(centers, centers->length, vec);
centers->length++; centers->length++;
@@ -160,7 +176,7 @@ QuickCenters(Relation index, VectorArray samples, VectorArray centers)
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf * https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/ */
static void static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers) ElkanKmeans(Relation index, List *samples, VectorArray centers)
{ {
FmgrInfo *procinfo; FmgrInfo *procinfo;
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
@@ -171,7 +187,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
int64 k; int64 k;
int dimensions = centers->dim; int dimensions = centers->dim;
int numCenters = centers->maxlen; int numCenters = centers->maxlen;
int numSamples = samples->length; int numSamples = list_length(samples);
VectorArray newCenters; VectorArray newCenters;
int *centerCounts; int *centerCounts;
int *closestCenters; int *closestCenters;
@@ -182,7 +198,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
float *newcdist; float *newcdist;
/* Calculate allocation sizes */ /* Calculate allocation sizes */
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->dim); Size samplesSize = 0;
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->dim); Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->dim);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, dimensions); Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, dimensions);
Size centerCountsSize = sizeof(int) * numCenters; Size centerCountsSize = sizeof(int) * numCenters;
@@ -326,7 +342,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
if (upperBound[j] <= halfcdist[closestCenters[j] * numCenters + k]) if (upperBound[j] <= halfcdist[closestCenters[j] * numCenters + k])
continue; continue;
vec = VectorArrayGet(samples, j); vec = list_nth(samples, j);
/* Step 3a */ /* Step 3a */
if (rj) if (rj)
@@ -377,7 +393,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
{ {
int closestCenter; int closestCenter;
vec = VectorArrayGet(samples, j); vec = list_nth(samples, j);
closestCenter = closestCenters[j]; closestCenter = closestCenters[j];
/* Increment sum and count of closest center */ /* Increment sum and count of closest center */
@@ -514,9 +530,9 @@ CheckCenters(Relation index, VectorArray centers)
* We use spherical k-means for inner product and cosine * We use spherical k-means for inner product and cosine
*/ */
void void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers) IvfflatKmeans(Relation index, List *samples, VectorArray centers)
{ {
if (samples->length <= centers->maxlen) if (list_length(samples) <= centers->maxlen)
QuickCenters(index, samples, centers); QuickCenters(index, samples, centers);
else else
ElkanKmeans(index, samples, centers); ElkanKmeans(index, samples, centers);