Compare commits

...

3 Commits

Author SHA1 Message Date
Andrew Kane
c599017441 Improved naming [skip ci] 2026-07-10 17:37:27 -07:00
Andrew Kane
f341cea329 Reduced memory usage for few samples for IVFFlat index builds [skip ci] 2026-07-10 17:33:05 -07:00
Andrew Kane
73356ecfa7 Improved check for VectorArrayInit [skip ci] 2026-07-10 16:30:35 -07:00
2 changed files with 35 additions and 6 deletions

View File

@@ -8,6 +8,7 @@
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
@@ -105,16 +106,44 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
}
/*
* Quick approach if we have no data
* Check if vector array contains a vector
*/
static bool
VectorArrayContains(VectorArray arr, Pointer val)
{
Datum d = PointerGetDatum(val);
for (int i = 0; i < arr->length; i++)
{
if (datumIsEqual(d, PointerGetDatum(VectorArrayGet(arr, i)), false, -1))
return true;
}
return false;
}
/*
* Quick approach if we have little data
*/
static void
RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
QuickCenters(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
{
int dimensions = centers->dim;
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
Oid collation = index->rd_indcollation[0];
float *x = (float *) palloc(sizeof(float) * dimensions);
/* Fill with unique samples (already normalized) */
for (int i = 0; i < samples->length; i++)
{
Pointer sample = VectorArrayGet(samples, i);
if (!VectorArrayContains(centers, sample))
{
VectorArraySet(centers, centers->length, sample);
centers->length++;
}
}
/* Fill with random data */
while (centers->length < centers->maxlen)
{
@@ -548,8 +577,8 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(kmeansCtx);
if (samples->length == 0)
RandomCenters(index, centers, typeInfo);
if (samples->length <= centers->maxlen)
QuickCenters(index, samples, centers, typeInfo);
else
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);

View File

@@ -25,8 +25,8 @@ VectorArrayInit(int maxlen, int dimensions, Size itemsize)
{
VectorArray res;
if (maxlen < 1 || dimensions < 1)
elog(ERROR, "safety check failed");
if (maxlen < 1 || dimensions < 1 || itemsize == 0)
elog(ERROR, "cannot create vector array");
/* Ensure items are aligned to prevent UB */
itemsize = MAXALIGN(itemsize);