Use FreeTupleDesc

.github/workflows/build.yml

@@ -8,6 +8,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - postgres: 17
+            os: ubuntu-22.04
           - postgres: 16
             os: ubuntu-22.04
           - postgres: 15
@@ -21,7 +23,7 @@ jobs:
           - postgres: 11
             os: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: ${{ matrix.postgres }}
@@ -43,7 +45,7 @@ jobs:
     runs-on: macos-latest
     if: ${{ !startsWith(github.ref_name, 'windows') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: 14
@@ -65,7 +67,7 @@ jobs:
     runs-on: windows-latest
     if: ${{ !startsWith(github.ref_name, 'mac') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ankane/setup-postgres@v1
         with:
           postgres-version: 14

src/ivfbuild.c

@@ -11,7 +11,6 @@
 #include "miscadmin.h"
 #include "storage/bufmgr.h"
 #include "tcop/tcopprot.h"
-#include "utils/datum.h"
 #include "utils/memutils.h"
 
 #if PG_VERSION_NUM >= 140000
@@ -66,18 +65,11 @@
 static void
 AddSample(Datum *values, IvfflatBuildState * buildstate)
 {
-	MemoryContext oldCtx;
-	Datum		value;
-	int			targsamples = buildstate->targsamples;
-
-	/* Use memory context since detoast can allocate */
-	oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
+	VectorArray samples = buildstate->samples;
+	int			targsamples = samples->maxlen;
 
 	/* Detoast once for all calls */
-	value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
-
-	/* Restore memory context */
-	MemoryContextSwitchTo(oldCtx);
+	Datum		value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
 
 	/*
 	 * Normalize with KMEANS_NORM_PROC since spherical distance function
@@ -89,23 +81,18 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
 			return;
 	}
 
-	/* Copy datum */
-	value = datumCopy(value, false, -1);
-
-	/* Reset memory context */
-	MemoryContextReset(buildstate->tmpCtx);
-
-	if (list_length(buildstate->samples) < targsamples)
-		buildstate->samples = lappend(buildstate->samples, DatumGetVector(value));
+	if (samples->length < targsamples)
+	{
+		VectorArraySet(samples, samples->length, DatumGetVector(value));
+		samples->length++;
+	}
 	else
 	{
 		if (buildstate->rowstoskip < 0)
-			buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, list_length(buildstate->samples), targsamples);
+			buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
 
 		if (buildstate->rowstoskip <= 0)
 		{
-			ListCell   *lc;
-
 #if PG_VERSION_NUM >= 150000
 			int			k = (int) (targsamples * sampler_random_fract(&buildstate->rstate.randstate));
 #else
@@ -113,8 +100,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
 #endif
 
 			Assert(k >= 0 && k < targsamples);
-			lc = list_nth_cell(buildstate->samples, k);
-			lfirst(lc) = DatumGetVector(value);
+			VectorArraySet(samples, k, DatumGetVector(value));
 		}
 
 		buildstate->rowstoskip -= 1;
@@ -129,13 +115,21 @@ SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
 			   bool *isnull, bool tupleIsAlive, void *state)
 {
 	IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
+	MemoryContext oldCtx;
 
 	/* Skip nulls */
 	if (isnull[0])
 		return;
 
+	/* Use memory context since detoast can allocate */
+	oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
+
 	/* Add sample */
-	AddSample(values, buildstate);
+	AddSample(values, state);
+
+	/* Reset memory context */
+	MemoryContextSwitchTo(oldCtx);
+	MemoryContextReset(buildstate->tmpCtx);
 }
 
 /*
@@ -144,7 +138,7 @@ SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
 static void
 SampleRows(IvfflatBuildState * buildstate)
 {
-	int			targsamples = buildstate->targsamples;
+	int			targsamples = buildstate->samples->maxlen;
 	BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
 
 	buildstate->rowstoskip = -1;
@@ -293,7 +287,7 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 #else
 	TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc);
 #endif
-	TupleDesc	tupdesc = RelationGetDescr(index);
+	TupleDesc	tupdesc = IvfflatTupleDesc(index);
 
 	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
 
@@ -345,6 +339,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 		/* Set the start and insert pages */
 		IvfflatUpdateList(index, buildstate->listInfo[i], insertPage, InvalidBlockNumber, startPage, forkNum);
 	}
+
+	FreeTupleDesc(tupdesc);
 }
 
 /*
@@ -455,13 +451,12 @@ ComputeCenters(IvfflatBuildState * buildstate)
 
 	/* Sample rows */
 	/* TODO Ensure within maintenance_work_mem */
-	buildstate->samples = NIL;
-	buildstate->targsamples = numSamples;
+	buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions);
 	if (buildstate->heap != NULL)
 	{
 		SampleRows(buildstate);
 
-		if (list_length(buildstate->samples) < buildstate->lists)
+		if (buildstate->samples->length < buildstate->lists)
 		{
 			ereport(NOTICE,
 					(errmsg("ivfflat index created with little data"),
@@ -474,7 +469,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
 	IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
 
 	/* Free samples before we allocate more memory */
-	list_free_deep(buildstate->samples);
+	VectorArrayFree(buildstate->samples);
 }
 
 /*

src/ivfflat.h

@@ -80,10 +80,6 @@
 #define RandomInt() random()
 #endif
 
-#if PG_VERSION_NUM < 130000
-#define list_sort(list, cmp) list_qsort(list, cmp)
-#endif
-
 /* Variables */
 extern int	ivfflat_probes;
 
@@ -182,8 +178,7 @@ typedef struct IvfflatBuildState
 	Oid			collation;
 
 	/* Variables */
-	List	   *samples;
-	int			targsamples;
+	VectorArray samples;
 	VectorArray centers;
 	ListInfo   *listInfo;
 	Vector	   *normvec;
@@ -279,7 +274,7 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
 VectorArray VectorArrayInit(int maxlen, int dimensions);
 void		VectorArrayFree(VectorArray arr);
 void		PrintVectorArray(char *msg, VectorArray arr);
-void		IvfflatKmeans(Relation index, List *samples, VectorArray centers);
+void		IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
 FmgrInfo   *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
 bool		IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
 int			IvfflatGetLists(Relation index);
@@ -292,6 +287,7 @@ void		IvfflatInitPage(Buffer buf, Page page);
 void		IvfflatInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state);
 void		IvfflatInit(void);
 PGDLLEXPORT void IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc);
+TupleDesc	IvfflatTupleDesc(Relation index);
 
 /* Index access methods */
 IndexBuildResult *ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo);

src/ivfinsert.c

@@ -94,7 +94,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
 	originalInsertPage = insertPage;
 
 	/* Form tuple */
-	itup = index_form_tuple(RelationGetDescr(index), &value, isnull);
+	itup = index_form_tuple(IvfflatTupleDesc(index), &value, isnull);
 	itup->t_tid = *heap_tid;
 
 	/* Get tuple size */

src/ivfkmeans.c

@@ -12,20 +12,20 @@
  * https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf
  */
 static void
-InitCenters(Relation index, List *samples, VectorArray centers, float *lowerBound)
+InitCenters(Relation index, VectorArray samples, VectorArray centers, float *lowerBound)
 {
 	FmgrInfo   *procinfo;
 	Oid			collation;
 	int64		j;
-	float	   *weight = palloc(list_length(samples) * sizeof(float));
+	float	   *weight = palloc(samples->length * sizeof(float));
 	int			numCenters = centers->maxlen;
-	int			numSamples = list_length(samples);
+	int			numSamples = samples->length;
 
 	procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
 	collation = index->rd_indcollation[0];
 
 	/* Choose an initial center uniformly at random */
-	VectorArraySet(centers, 0, list_nth(samples, RandomInt() % list_length(samples)));
+	VectorArraySet(centers, 0, VectorArrayGet(samples, RandomInt() % samples->length));
 	centers->length++;
 
 	for (j = 0; j < numSamples; j++)
@@ -42,7 +42,7 @@ InitCenters(Relation index, List *samples, VectorArray centers, float *lowerBoun
 
 		for (j = 0; j < numSamples; j++)
 		{
-			Vector	   *vec = list_nth(samples, j);
+			Vector	   *vec = VectorArrayGet(samples, j);
 			double		distance;
 
 			/* Only need to compute distance for new center */
@@ -74,7 +74,7 @@ InitCenters(Relation index, List *samples, VectorArray centers, float *lowerBoun
 				break;
 		}
 
-		VectorArraySet(centers, i + 1, list_nth(samples, j));
+		VectorArraySet(centers, i + 1, VectorArrayGet(samples, j));
 		centers->length++;
 	}
 
@@ -106,41 +106,25 @@ CompareVectors(const void *a, const void *b)
 	return vector_cmp_internal((Vector *) a, (Vector *) b);
 }
 
-/*
- * Compare list vectors
- */
-static int
-#if PG_VERSION_NUM >= 130000
-CompareListVectors(const ListCell *a, const ListCell *b)
-#else
-CompareListVectors(const void *a, const void *b)
-#endif
-{
-	Vector	   *va = lfirst((ListCell *) a);
-	Vector	   *vb = lfirst((ListCell *) b);
-
-	return CompareVectors(va, vb);
-}
-
 /*
  * Quick approach if we have little data
  */
 static void
-QuickCenters(Relation index, List *samples, VectorArray centers)
+QuickCenters(Relation index, VectorArray samples, VectorArray centers)
 {
 	int			dimensions = centers->dim;
 	Oid			collation = index->rd_indcollation[0];
 	FmgrInfo   *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
 
 	/* Copy existing vectors while avoiding duplicates */
-	if (list_length(samples) > 0)
+	if (samples->length > 0)
 	{
-		list_sort(samples, CompareListVectors);
-		for (int i = 0; i < list_length(samples); i++)
+		qsort(samples->items, samples->length, VECTOR_SIZE(samples->dim), CompareVectors);
+		for (int i = 0; i < samples->length; i++)
 		{
-			Vector	   *vec = list_nth(samples, i);
+			Vector	   *vec = VectorArrayGet(samples, i);
 
-			if (i == 0 || CompareVectors(vec, list_nth(samples, i - 1)) != 0)
+			if (i == 0 || CompareVectors(vec, VectorArrayGet(samples, i - 1)) != 0)
 			{
 				VectorArraySet(centers, centers->length, vec);
 				centers->length++;
@@ -176,7 +160,7 @@ QuickCenters(Relation index, List *samples, VectorArray centers)
  * https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
  */
 static void
-ElkanKmeans(Relation index, List *samples, VectorArray centers)
+ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 {
 	FmgrInfo   *procinfo;
 	FmgrInfo   *normprocinfo;
@@ -187,7 +171,7 @@ ElkanKmeans(Relation index, List *samples, VectorArray centers)
 	int64		k;
 	int			dimensions = centers->dim;
 	int			numCenters = centers->maxlen;
-	int			numSamples = list_length(samples);
+	int			numSamples = samples->length;
 	VectorArray newCenters;
 	int		   *centerCounts;
 	int		   *closestCenters;
@@ -198,7 +182,7 @@ ElkanKmeans(Relation index, List *samples, VectorArray centers)
 	float	   *newcdist;
 
 	/* Calculate allocation sizes */
-	Size		samplesSize = 0;
+	Size		samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->dim);
 	Size		centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->dim);
 	Size		newCentersSize = VECTOR_ARRAY_SIZE(numCenters, dimensions);
 	Size		centerCountsSize = sizeof(int) * numCenters;
@@ -342,7 +326,7 @@ ElkanKmeans(Relation index, List *samples, VectorArray centers)
 				if (upperBound[j] <= halfcdist[closestCenters[j] * numCenters + k])
 					continue;
 
-				vec = list_nth(samples, j);
+				vec = VectorArrayGet(samples, j);
 
 				/* Step 3a */
 				if (rj)
@@ -393,7 +377,7 @@ ElkanKmeans(Relation index, List *samples, VectorArray centers)
 		{
 			int			closestCenter;
 
-			vec = list_nth(samples, j);
+			vec = VectorArrayGet(samples, j);
 			closestCenter = closestCenters[j];
 
 			/* Increment sum and count of closest center */
@@ -530,9 +514,9 @@ CheckCenters(Relation index, VectorArray centers)
  * We use spherical k-means for inner product and cosine
  */
 void
-IvfflatKmeans(Relation index, List *samples, VectorArray centers)
+IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers)
 {
-	if (list_length(samples) <= centers->maxlen)
+	if (samples->length <= centers->maxlen)
 		QuickCenters(index, samples, centers);
 	else
 		ElkanKmeans(index, samples, centers);

src/ivfutils.c

@@ -4,6 +4,10 @@
 #include "storage/bufmgr.h"
 #include "vector.h"
 
+#if PG_VERSION_NUM < 130000
+#define TYPSTORAGE_PLAIN 'p'
+#endif
+
 /*
  * Allocate a vector array
  */
@@ -238,3 +242,17 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
 		UnlockReleaseBuffer(buf);
 	}
 }
+
+/*
+ * Get the tuple descriptor
+ */
+TupleDesc
+IvfflatTupleDesc(Relation index)
+{
+	TupleDesc	tupdesc = CreateTupleDescCopyConstr(RelationGetDescr(index));
+
+	/* Prevent compression */
+	TupleDescAttr(tupdesc, 0)->attstorage = TYPSTORAGE_PLAIN;
+
+	return tupdesc;
+}