liuyu/pgvector
1
0
Fork 0
mirror of https://github.com/pgvector/pgvector.git synced 2026-07-14 08:36:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed index creation exceeding maintenance_work_mem and fixed error when lists > 1600

Browse Source
This commit is contained in:
Andrew Kane
2022-02-11 03:10:19 -08:00
parent 0094b70a36
commit cab103a2e7
2 changed files with 14 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@@ -1,6 +1,8 @@
## 0.2.5 (unreleased) ## 0.2.5 (unreleased)
- Reduced memory usage during index creation - Reduced memory usage during index creation
- Fixed index creation exceeding `maintenance_work_mem`
- Fixed error when lists > 1600
## 0.2.4 (2022-02-06) ## 0.2.4 (2022-02-06)

13
src/ivfkmeans.c
View File

@@ -190,6 +190,17 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
bool rjreset; bool rjreset;
double dxcx; double dxcx;
double dxc; double dxc;
Size lowerBoundSize = sizeof(float) * numSamples * numCenters;
/* TODO Add other allocations */
Size totalSize = lowerBoundSize;
/* Check memory requirements */
if (totalSize / 1024 > maintenance_work_mem)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %zu MB",
totalSize / (1024 * 1024), ((Size) maintenance_work_mem) / 1024)));
/* Set support functions */ /* Set support functions */
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC); procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
@@ -200,7 +211,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
/* Use float instead of double to save memory */ /* Use float instead of double to save memory */
centerCounts = palloc(sizeof(int) * numCenters); centerCounts = palloc(sizeof(int) * numCenters);
closestCenters = palloc(sizeof(int) * numSamples); closestCenters = palloc(sizeof(int) * numSamples);
lowerBound = palloc(sizeof(float) * numSamples * numCenters); lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
upperBound = palloc(sizeof(float) * numSamples); upperBound = palloc(sizeof(float) * numSamples);
s = palloc(sizeof(float) * numCenters); s = palloc(sizeof(float) * numCenters);
halfcdist = palloc(sizeof(float) * numCenters * numCenters); halfcdist = palloc(sizeof(float) * numCenters * numCenters);