Fixed version check for palloc variants, take 2

Co-authored-by: 0xJi3F

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.8.6 (unreleased)
+
+- Fixed array to `sparsevec` cast not limiting non-zero elements
+- Fixed memory usage for IVFFlat index scans with nested loop joins
+
 ## 0.8.5 (2026-07-08)
 
 - Reduced memory usage for small tables for IVFFlat index builds

README.md

@@ -465,6 +465,16 @@ If filtering by many different values, consider [partitioning](https://www.postg
 CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
 ```
 
+## Multitenancy
+
+For applications with multiple tenants, sharing an approximate index between tenants means vectors from one tenant can affect recall (and speed) for other tenants.
+
+For tenant isolation, use [list partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) or separate tables.
+
+```sql
+CREATE TABLE items (customer_id int, embedding vector(3)) PARTITION BY LIST(customer_id);
+```
+
 ## Iterative Index Scans
 
 With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
@@ -1223,7 +1233,7 @@ Note: Replace `18` with your Postgres server version
 Install the FreeBSD package with:
 
 ```sh
-pkg install postgresql17-pgvector
+pkg install postgresql18-pgvector
 ```
 
 or the port with:

src/halfvec.c

@@ -27,8 +27,12 @@
 #include "parser/scansup.h"
 #endif
 
+#if PG_VERSION_NUM < 140006
+#define palloc_array(type, count) ((type *) palloc(sizeof(type) * (count)))
+#endif
+
 #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
-#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
+#define CreateStateDatums(dim) palloc_array(Datum, (dim) + 1)
 
 /*
  * Get a half from a message buffer
@@ -513,7 +517,7 @@ halfvec_to_float4(PG_FUNCTION_ARGS)
 	Datum	   *datums;
 	ArrayType  *result;
 
-	datums = (Datum *) palloc(sizeof(Datum) * vec->dim);
+	datums = palloc_array(Datum, vec->dim);
 
 	for (int i = 0; i < vec->dim; i++)
 		datums[i] = Float4GetDatum(HalfToFloat4(vec->x[i]));

src/hnsw.h

@@ -105,6 +105,12 @@ typedef Pointer Item;
 #define SeedRandom(seed) srandom(seed)
 #endif
 
+#if PG_VERSION_NUM < 140006
+#define palloc_object(type) ((type *) palloc(sizeof(type)))
+#define palloc0_object(type) ((type *) palloc0(sizeof(type)))
+#define palloc_array(type, count) ((type *) palloc(sizeof(type) * (count)))
+#endif
+
 #define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
 #define HnswIsNeighborTuple(tup) ((tup)->type == HNSW_NEIGHBOR_TUPLE_TYPE)
 

src/hnswbuild.c

@@ -930,7 +930,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
 	Size		estother;
 	HnswShared *hnswshared;
 	char	   *hnswarea;
-	HnswLeader *hnswleader = (HnswLeader *) palloc0(sizeof(HnswLeader));
+	HnswLeader *hnswleader = palloc0_object(HnswLeader);
 	bool		leaderparticipates = true;
 	int			querylen;
 
@@ -1151,7 +1151,7 @@ hnswbuild(Relation heap, Relation index, IndexInfo *indexInfo)
 
 	BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
 
-	result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
+	result = palloc_object(IndexBuildResult);
 	result->heap_tuples = buildstate.reltuples;
 	result->index_tuples = buildstate.indtuples;
 

src/hnswscan.c

@@ -136,7 +136,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
 
 	scan = RelationGetIndexScan(index, nkeys, norderbys);
 
-	so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
+	so = palloc_object(HnswScanOpaqueData);
 	so->typeInfo = HnswGetTypeInfo(index);
 
 	/* Set support functions */

src/hnswutils.c

@@ -282,7 +282,7 @@ HnswAddHeapTid(HnswElement element, ItemPointer heaptid)
 HnswElement
 HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
 {
-	HnswElement element = palloc(sizeof(HnswElementData));
+	HnswElement element = palloc_object(HnswElementData);
 	char	   *base = NULL;
 
 	element->blkno = blkno;
@@ -596,7 +596,7 @@ GetElementDistance(char *base, HnswElement element, HnswQuery * q, HnswSupport *
 static HnswSearchCandidate *
 HnswInitSearchCandidate(char *base, HnswElement element, double distance)
 {
-	HnswSearchCandidate *sc = palloc(sizeof(HnswSearchCandidate));
+	HnswSearchCandidate *sc = palloc_object(HnswSearchCandidate);
 
 	HnswPtrStore(base, sc->element, element);
 	sc->distance = distance;
@@ -831,7 +831,7 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
 	HnswNeighborArray *localNeighborhood = NULL;
 	Size		neighborhoodSize = 0;
 	int			lm = HnswGetLayerM(m, lc);
-	HnswUnvisited *unvisited = palloc(lm * sizeof(HnswUnvisited));
+	HnswUnvisited *unvisited = palloc_array(HnswUnvisited, lm);
 	int			unvisitedLength;
 	bool		inMemory = index == NULL;
 
@@ -1074,7 +1074,7 @@ SelectNeighbors(char *base, List *c, int lm, HnswSupport * support, bool *closer
 	if (list_length(w) <= lm)
 		return w;
 
-	wd = palloc(sizeof(HnswCandidate *) * list_length(w));
+	wd = palloc_array(HnswCandidate *, list_length(w));
 
 	/* Ensure order of candidates is deterministic for closer caching */
 	if (sortCandidates)
@@ -1328,7 +1328,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
 		foreach(lc2, w)
 		{
 			HnswSearchCandidate *sc = lfirst(lc2);
-			HnswCandidate *hc = palloc(sizeof(HnswCandidate));
+			HnswCandidate *hc = palloc_object(HnswCandidate);
 
 			hc->element = sc->element;
 			hc->distance = sc->distance;

src/hnswvacuum.c

@@ -737,7 +737,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
 	Relation	index = info->index;
 
 	if (stats == NULL)
-		stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult));
+		stats = palloc0_object(IndexBulkDeleteResult);
 
 	vacuumstate->index = index;
 	vacuumstate->stats = stats;

src/ivfbuild.c

@@ -396,7 +396,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 	buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
 
 	/* TODO Move allocation to page creation */
-	buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
+	buildstate->listInfo = palloc_array(ListInfo, buildstate->lists);
 
 	buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
 											   "Ivfflat build temporary context",
@@ -404,8 +404,8 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 
 #ifdef IVFFLAT_KMEANS_DEBUG
 	buildstate->inertia = 0;
-	buildstate->listSums = palloc0(sizeof(double) * buildstate->lists);
+	buildstate->listSums = palloc0_array(double, buildstate->lists);
-	buildstate->listCounts = palloc0(sizeof(int) * buildstate->lists);
+	buildstate->listCounts = palloc0_array(int, buildstate->lists);
 #endif
 
 	buildstate->ivfleader = NULL;
@@ -662,7 +662,7 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
 	IndexInfo  *indexInfo;
 
 	/* Initialize local tuplesort coordination state */
-	coordinate = palloc0(sizeof(SortCoordinateData));
+	coordinate = palloc0_object(SortCoordinateData);
 	coordinate->isWorker = true;
 	coordinate->nParticipants = -1;
 	coordinate->sharedsort = sharedsort;
@@ -757,7 +757,7 @@ IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc)
 	indexRel = index_open(ivfshared->indexrelid, indexLockmode);
 
 	/* Initialize worker's own spool */
-	ivfspool = (IvfflatSpool *) palloc0(sizeof(IvfflatSpool));
+	ivfspool = palloc0_object(IvfflatSpool);
 	ivfspool->heap = heapRel;
 	ivfspool->index = indexRel;
 
@@ -812,7 +812,7 @@ IvfflatLeaderParticipateAsWorker(IvfflatBuildState * buildstate)
 	int			sortmem;
 
 	/* Allocate memory and initialize private spool */
-	leaderworker = (IvfflatSpool *) palloc0(sizeof(IvfflatSpool));
+	leaderworker = palloc0_object(IvfflatSpool);
 	leaderworker->heap = buildstate->heap;
 	leaderworker->index = buildstate->index;
 
@@ -838,7 +838,7 @@ IvfflatBeginParallel(IvfflatBuildState * buildstate, bool isconcurrent, int requ
 	IvfflatShared *ivfshared;
 	Sharedsort *sharedsort;
 	char	   *ivfcenters;
-	IvfflatLeader *ivfleader = (IvfflatLeader *) palloc0(sizeof(IvfflatLeader));
+	IvfflatLeader *ivfleader = palloc0_object(IvfflatLeader);
 	bool		leaderparticipates = true;
 	int			querylen;
 
@@ -987,7 +987,7 @@ AssignTuples(IvfflatBuildState * buildstate)
 	/* Set up coordination state if at least one worker launched */
 	if (buildstate->ivfleader)
 	{
-		coordinate = (SortCoordinate) palloc0(sizeof(SortCoordinateData));
+		coordinate = palloc0_object(SortCoordinateData);
 		coordinate->isWorker = false;
 		coordinate->nParticipants = buildstate->ivfleader->nparticipanttuplesorts;
 		coordinate->sharedsort = buildstate->ivfleader->sharedsort;
@@ -1072,7 +1072,7 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
 
 	BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
 
-	result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
+	result = palloc_object(IndexBuildResult);
 	result->heap_tuples = buildstate.reltuples;
 	result->index_tuples = buildstate.indtuples;
 

src/ivfflat.h

@@ -89,6 +89,13 @@ typedef Pointer Item;
 #define SeedRandom(seed) srandom(seed)
 #endif
 
+#if PG_VERSION_NUM < 140006
+#define palloc_object(type) ((type *) palloc(sizeof(type)))
+#define palloc0_object(type) ((type *) palloc0(sizeof(type)))
+#define palloc_array(type, count) ((type *) palloc(sizeof(type) * (count)))
+#define palloc0_array(type, count) ((type *) palloc0(sizeof(type) * (count)))
+#endif
+
 /* Variables */
 extern int	ivfflat_probes;
 extern int	ivfflat_iterative_scan;
@@ -305,7 +312,7 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
 static inline Pointer
 VectorArrayGet(VectorArray arr, int offset)
 {
-	if (offset >= arr->maxlen)
+	if (offset < 0 || offset >= arr->maxlen)
 		elog(ERROR, "safety check failed");
 
 	return ((char *) arr->items) + (offset * arr->itemsize);

src/ivfkmeans.c

@@ -26,7 +26,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
 	FmgrInfo   *procinfo;
 	Oid			collation;
 	int64		j;
-	float	   *weight = palloc(samples->length * sizeof(float));
+	float	   *weight = palloc_array(float, samples->length);
 	int			numCenters = centers->maxlen;
 	int			numSamples = samples->length;
 
@@ -113,7 +113,7 @@ RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeI
 	int			dimensions = centers->dim;
 	FmgrInfo   *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
 	Oid			collation = index->rd_indcollation[0];
-	float	   *x = (float *) palloc(sizeof(float) * dimensions);
+	float	   *x = palloc_array(float, dimensions);
 
 	/* Fill with random data */
 	while (centers->length < centers->maxlen)
@@ -480,7 +480,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
 static void
 CheckElements(VectorArray centers, const IvfflatTypeInfo * typeInfo)
 {
-	float	   *scratch = palloc(sizeof(float) * centers->dim);
+	float	   *scratch = palloc_array(float, centers->dim);
 
 	for (int i = 0; i < centers->length; i++)
 	{

src/ivfscan.c

@@ -276,12 +276,13 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	if (maxProbes > lists)
 		maxProbes = lists;
 
-	so = (IvfflatScanOpaque) palloc(sizeof(IvfflatScanOpaqueData));
+	so = palloc_object(IvfflatScanOpaqueData);
 	so->typeInfo = IvfflatGetTypeInfo(index);
 	so->first = true;
 	so->probes = probes;
 	so->maxProbes = maxProbes;
 	so->dimensions = dimensions;
+	so->value = PointerGetDatum(NULL);
 
 	/* Set support functions */
 	so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
@@ -317,9 +318,9 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
 	so->bas = GetAccessStrategy(BAS_BULKREAD);
 
 	so->listQueue = pairingheap_allocate(CompareLists, scan);
-	so->listPages = palloc(maxProbes * sizeof(BlockNumber));
+	so->listPages = palloc_array(BlockNumber, maxProbes);
 	so->listIndex = 0;
-	so->lists = palloc(maxProbes * sizeof(IvfflatScanList));
+	so->lists = palloc_array(IvfflatScanList, maxProbes);
 
 	MemoryContextSwitchTo(oldCtx);
 
@@ -340,6 +341,12 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
 	pairingheap_reset(so->listQueue);
 	so->listIndex = 0;
 
+	if (so->normprocinfo != NULL && DatumGetPointer(so->value) != NULL)
+	{
+		pfree(DatumGetPointer(so->value));
+		so->value = PointerGetDatum(NULL);
+	}
+
 	if (keys && scan->numberOfKeys > 0)
 		memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
 

src/ivfutils.c

@@ -25,13 +25,13 @@ VectorArrayInit(int maxlen, int dimensions, Size itemsize)
 {
 	VectorArray res;
 
-	if (maxlen < 1 || dimensions < 1)
+	if (maxlen < 1 || dimensions < 1 || itemsize == 0)
-		elog(ERROR, "safety check failed");
+		elog(ERROR, "cannot create vector array");
 
 	/* Ensure items are aligned to prevent UB */
 	itemsize = MAXALIGN(itemsize);
 
-	res = palloc(sizeof(VectorArrayData));
+	res = palloc_object(VectorArrayData);
 	res->length = 0;
 	res->maxlen = maxlen;
 	res->dim = dimensions;

src/ivfvacuum.c

@@ -24,7 +24,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 	BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
 
 	if (stats == NULL)
-		stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult));
+		stats = palloc0_object(IndexBulkDeleteResult);
 
 	/* Iterate over list pages */
 	while (BlockNumberIsValid(blkno))

src/sparsevec.c

@@ -26,6 +26,10 @@
 #include "parser/scansup.h"
 #endif
 
+#if PG_VERSION_NUM < 140006
+#define palloc_array(type, count) ((type *) palloc(sizeof(type) * (count)))
+#endif
+
 typedef struct SparseInputElement
 {
 	int32		index;
@@ -223,7 +227,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("sparsevec cannot have more than %d non-zero elements", SPARSEVEC_MAX_NNZ)));
 
-	elements = palloc(maxNnz * sizeof(SparseInputElement));
+	elements = palloc_array(SparseInputElement, maxNnz);
 
 	pt = lit;
 
@@ -614,6 +618,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
 			nnz++;
 	}
 
+	CheckNnz(nnz, dim);
 	result = InitSparseVector(dim, nnz);
 	values = SPARSEVEC_VALUES(result);
 	for (int i = 0; i < dim; i++)
@@ -657,6 +662,7 @@ halfvec_to_sparsevec(PG_FUNCTION_ARGS)
 			nnz++;
 	}
 
+	CheckNnz(nnz, dim);
 	result = InitSparseVector(dim, nnz);
 	values = SPARSEVEC_VALUES(result);
 	for (int i = 0; i < dim; i++)
@@ -745,6 +751,7 @@ array_to_sparsevec(PG_FUNCTION_ARGS)
 				 errmsg("unsupported array type")));
 	}
 
+	CheckNnz(nnz, nelemsp);
 	result = InitSparseVector(nelemsp, nnz);
 	values = SPARSEVEC_VALUES(result);
 

src/vector.c

@@ -30,8 +30,12 @@
 #include "parser/scansup.h"
 #endif
 
+#if PG_VERSION_NUM < 140006
+#define palloc_array(type, count) ((type *) palloc(sizeof(type) * (count)))
+#endif
+
 #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
-#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
+#define CreateStateDatums(dim) palloc_array(Datum, (dim) + 1)
 
 #if defined(USE_TARGET_CLONES) && !defined(__FMA__)
 #define VECTOR_TARGET_CLONES __attribute__((target_clones("default", "fma")))
@@ -516,7 +520,7 @@ vector_to_float4(PG_FUNCTION_ARGS)
 	Datum	   *datums;
 	ArrayType  *result;
 
-	datums = (Datum *) palloc(sizeof(Datum) * vec->dim);
+	datums = palloc_array(Datum, vec->dim);
 
 	for (int i = 0; i < vec->dim; i++)
 		datums[i] = Float4GetDatum(vec->x[i]);

test/expected/cast.out

@@ -268,6 +268,10 @@ SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
 ERROR:  vector cannot have more than 16000 dimensions
 SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
 ERROR:  vector cannot have more than 16000 dimensions
+SELECT array_agg(n)::halfvec FROM generate_series(1, 16001) n;
+ERROR:  halfvec cannot have more than 16000 dimensions
+SELECT array_agg(n)::sparsevec FROM generate_series(1, 16001) n;
+ERROR:  sparsevec cannot have more than 16000 non-zero elements
 -- ensure no error
 SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
  ?column? 

test/sql/cast.sql

@@ -76,6 +76,8 @@ SELECT '{{1}}'::real[]::sparsevec;
 
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
 SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
+SELECT array_agg(n)::halfvec FROM generate_series(1, 16001) n;
+SELECT array_agg(n)::sparsevec FROM generate_series(1, 16001) n;
 
 -- ensure no error
 SELECT ARRAY[1,2,3] = ARRAY[1,2,3];