Version bump to 0.3.2 [skip ci]

.github/workflows/build.yml

@@ -16,8 +16,9 @@ jobs:
     - uses: ankane/setup-postgres@v1
       with:
         postgres-version: ${{ matrix.postgres }}
+        dev-files: true
     - if: ${{ startsWith(matrix.os, 'ubuntu') }}
-      run: sudo apt-get update && sudo apt-get install postgresql-server-dev-${{ matrix.postgres }} libipc-run-perl
+      run: sudo apt-get update && sudo apt-get install libipc-run-perl
     - run: make
     - if: ${{ startsWith(matrix.os, 'ubuntu') }}
       run: |

.gitignore

@@ -5,3 +5,4 @@
 regression.*
 *.o
 *.so
+*.bc

CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.3.2 (2022-11-22)
+
+- Fixed `invalid memory alloc request size` error
+
+## 0.3.1 (2022-11-02)
+
+If upgrading from 0.2.7 or 0.3.0, [recreate](https://github.com/pgvector/pgvector#031) all `ivfflat` indexes after upgrading to ensure all data is indexed.
+
+- Fixed issue with inserts silently corrupting `ivfflat` indexes (introduced in 0.2.7)
+- Fixed segmentation fault with index creation when lists > 6500
+
 ## 0.3.0 (2022-10-15)
 
 - Added support for Postgres 15

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.3.0",
+	"version": "0.3.2",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -12,7 +12,7 @@
 	"prereqs": {
 		"runtime": {
 			"requires": {
-				"PostgreSQL": "10.0"
+				"PostgreSQL": "10.0.0"
 			}
 		}
 	},
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.3.0",
+			"version": "0.3.2",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.3.0
+EXTVERSION = 0.3.2
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)

README.md

@@ -17,7 +17,7 @@ Supports L2 distance, inner product, and cosine distance
 Compile and install the extension (supports Postgres 10+)
 
 ```sh
-git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git
+git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -232,7 +232,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
 You can also build the image manually
 
 ```sh
-git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git
+git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build -t pgvector .
 ```
@@ -270,6 +270,22 @@ Install the latest version and run:
 ALTER EXTENSION vector UPDATE;
 ```
 
+## Upgrade Notes
+
+### 0.3.1
+
+If upgrading from 0.2.7 or 0.3.0, recreate all `ivfflat` indexes after upgrading to ensure all data is indexed.
+
+```sql
+-- Postgres 12+
+REINDEX INDEX CONCURRENTLY index_name;
+
+-- Postgres < 12
+CREATE INDEX CONCURRENTLY temp_name ON table USING ivfflat (column opclass);
+DROP INDEX CONCURRENTLY index_name;
+ALTER INDEX temp_name RENAME TO index_name;
+```
+
 ## Thanks
 
 Thanks to:

sql/vector--0.3.0--0.3.1.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.1'" to load this file. \quit

sql/vector--0.3.1--0.3.2.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.2'" to load this file. \quit

src/ivfbuild.c

@@ -233,8 +233,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 	GenericXLogState *state;
 	int			list;
 	IndexTuple	itup = NULL;	/* silence compiler warning */
-	BlockNumber startPage = InvalidBlockNumber;
-	BlockNumber insertPage = InvalidBlockNumber;
+	BlockNumber startPage;
+	BlockNumber insertPage;
 	Size		itemsz;
 	int			i;
 	int64		inserted = 0;

src/ivfinsert.c

@@ -97,26 +97,42 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
 		}
 		else
 		{
-			/* Add a new page */
-			Buffer		newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
-			Page		newpage = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE);
+			Buffer		metabuf;
+			Buffer		newbuf;
+			Page		newpage;
 
+			/*
+			 * From ReadBufferExtended: Caller is responsible for ensuring
+			 * that only one backend tries to extend a relation at the same
+			 * time!
+			 */
+			metabuf = ReadBuffer(rel, IVFFLAT_METAPAGE_BLKNO);
+			LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE);
+
+			/* Add a new page */
+			newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
+			newpage = GenericXLogRegisterBuffer(state, newbuf, GENERIC_XLOG_FULL_IMAGE);
+
+			/* Init new page */
+			IvfflatInitPage(newbuf, newpage);
+
+			/* Update insert page */
 			insertPage = BufferGetBlockNumber(newbuf);
 
 			/* Update previous buffer */
 			IvfflatPageGetOpaque(page)->nextblkno = insertPage;
 
-			/* Init page */
-			IvfflatInitPage(newbuf, newpage);
-
 			/* Commit */
-			MarkBufferDirty(buf);
 			MarkBufferDirty(newbuf);
+			MarkBufferDirty(buf);
 			GenericXLogFinish(state);
 
-			/* Unlock */
-			UnlockReleaseBuffer(buf);
+			/* Unlock extend relation lock as early as possible */
+			UnlockReleaseBuffer(metabuf);
+
+			/* Unlock rest */
 			UnlockReleaseBuffer(newbuf);
+			UnlockReleaseBuffer(buf);
 		}
 	}
 

src/ivfkmeans.c

@@ -26,7 +26,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
 	FmgrInfo   *procinfo;
 	Oid			collation;
 	int			i;
-	int			j;
+	int64		j;
 	double		distance;
 	double		sum;
 	double		choice;
@@ -182,8 +182,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 	Vector	   *vec;
 	Vector	   *newCenter;
 	int			iteration;
-	int			j;
-	int			k;
+	int64		j;
+	int64		k;
 	int			dimensions = centers->dim;
 	int			numCenters = centers->maxlen;
 	int			numSamples = samples->length;
@@ -227,6 +227,10 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 				 errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
 						totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
 
+	/* Ensure indexing does not overflow */
+	if (numCenters * numCenters > INT_MAX)
+		elog(ERROR, "Indexing overflow detected. Please report a bug.");
+
 	/* Set support functions */
 	procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
 	normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
@@ -239,7 +243,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 	lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
 	upperBound = palloc(upperBoundSize);
 	s = palloc(sSize);
-	halfcdist = palloc(halfcdistSize);
+	halfcdist = palloc_extended(halfcdistSize, MCXT_ALLOC_HUGE);
 	newcdist = palloc(newcdistSize);
 
 	newCenters = VectorArrayInit(numCenters, dimensions);
@@ -259,8 +263,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 		minDistance = DBL_MAX;
 		closestCenter = -1;
 
-		vec = VectorArrayGet(samples, j);
-
 		/* Find closest center */
 		for (k = 0; k < numCenters; k++)
 		{

src/ivfutils.c

@@ -10,7 +10,7 @@
 VectorArray
 VectorArrayInit(int maxlen, int dimensions)
 {
-	VectorArray res = palloc0(VECTOR_ARRAY_SIZE(maxlen, dimensions));
+	VectorArray res = palloc_extended(VECTOR_ARRAY_SIZE(maxlen, dimensions), MCXT_ALLOC_ZERO | MCXT_ALLOC_HUGE);
 
 	res->length = 0;
 	res->maxlen = maxlen;

test/t/001_wal.pl

@@ -51,6 +51,11 @@ sub test_index_replay
 	return;
 }
 
+# Use ARRAY[random(), random(), random(), ...] over
+# SELECT array_agg(random()) FROM generate_series(1, $dim)
+# to generate different values for each row
+my $array_sql = join(",", ('random()') x $dim);
+
 # Initialize primary node
 $node_primary = get_new_node('primary');
 $node_primary->init(allows_streaming => 1);
@@ -74,7 +79,7 @@ $node_replica->start;
 $node_primary->safe_psql("postgres", "CREATE EXTENSION vector;");
 $node_primary->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
 $node_primary->safe_psql("postgres",
-	"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series(1, 100000) i;"
+	"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
 );
 $node_primary->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
 
@@ -90,7 +95,7 @@ for my $i (1 .. 10)
 	test_index_replay("vacuum $i");
 	my ($start, $end) = (100001 + ($i - 1) * 10000, 100000 + $i * 10000);
 	$node_primary->safe_psql("postgres",
-		"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series($start, $end) i;"
+		"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series($start, $end) i;"
 	);
 	test_index_replay("insert $i");
 }

test/t/007_inserts.pl

@@ -2,7 +2,11 @@ use strict;
 use warnings;
 use PostgresNode;
 use TestLib;
-use Test::More tests => 3;
+use Test::More tests => 5;
+
+my $dim = 768;
+
+my $array_sql = join(",", ('random()') x $dim);
 
 # Initialize node
 my $node = get_new_node('node');
@@ -11,9 +15,9 @@ $node->start;
 
 # Create table and index
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
-$node->safe_psql("postgres", "CREATE TABLE tst (v vector(768));");
+$node->safe_psql("postgres", "CREATE TABLE tst (v vector($dim));");
 $node->safe_psql("postgres",
-	"INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10000) i;"
+	"INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
 );
 $node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
 
@@ -24,10 +28,18 @@ $node->pgbench(
 	[qr{^$}],
 	"concurrent INSERTs",
 	{
-		"007_concurrent" => q(
-			BEGIN;
-			INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10) i;
-			COMMIT;
-		),
+		"007_inserts" => "INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10) i;"
 	}
 );
+
+my $expected = 10000 + 5 * 100 * 10;
+
+my $count = $node->safe_psql("postgres", "SELECT COUNT(*) FROM tst;");
+is($count, $expected);
+
+$count = $node->safe_psql("postgres", qq(
+	SET enable_seqscan = off;
+	SET ivfflat.probes = 100;
+	SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
+));
+is($count, $expected);

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat access method'
-default_version = '0.3.0'
+default_version = '0.3.2'
 module_pathname = '$libdir/vector'
 relocatable = true