Use fabs [skip ci]

CHANGELOG.md

@@ -1,7 +1,6 @@
-## 0.4.2 (2023-05-13)
+## 0.4.2 (unreleased)
 
 - Added notice when index created with little data
-- Fixed dimensions check for some direct function calls
 - Fixed installation error with Postgres 12.0-12.2
 
 ## 0.4.1 (2023-03-21)

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.4.2",
+	"version": "0.4.1",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.4.2",
+			"version": "0.4.1",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.4.2
+EXTVERSION = 0.4.1
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)
@@ -63,9 +63,3 @@ dist:
 
 docker:
 	docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest .
-
-.PHONY: docker-release
-
-docker-release:
-	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 -t ankane/pgvector:latest .
-	docker buildx build --push --platform linux/amd64,linux/arm64 -t ankane/pgvector:v$(EXTVERSION) .

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.4.2
+EXTVERSION = 0.4.1
 
 OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
 

README.md

@@ -16,7 +16,7 @@ Compile and install the extension (supports Postgres 11+)
 
 ```sh
 cd /tmp
-git clone --branch v0.4.2 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -307,7 +307,10 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
 
 #### What if I want to index vectors with more than 2,000 dimensions?
 
-You’ll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
+Two things you can try are:
+
+1. use dimensionality reduction
+2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
 
 #### Why am I seeing less results after adding an index?
 
@@ -355,11 +358,7 @@ If your machine has multiple Postgres installations, specify the path to [pg_con
 export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
 ```
 
-Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
+Then re-run the installation instructions (run `make clean` before `make` if needed)
-
-```sh
-sudo --preserve-env=PG_CONFIG make install
-```
 
 ### Missing Header
 
@@ -379,7 +378,7 @@ Support for Windows is currently experimental. Use `nmake` to build:
 
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\15"
-git clone --branch v0.4.2 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -400,7 +399,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.4.2 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build -t pgvector .
 ```
@@ -413,8 +412,6 @@ With Homebrew Postgres, you can use:
 brew install pgvector
 ```
 
-Note: This only adds it to the `postgresql@14` formula
-
 ### PGXN
 
 Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) with:
@@ -451,9 +448,10 @@ pgvector is available on [these providers](https://github.com/pgvector/pgvector/
 
 To request a new extension on other providers:
 
+- Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
 - Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
 - Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
-- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/managed-database/p/pgvector-extension-for-postgresql)
+- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
 - Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
 
 ## Upgrading

sql/vector--0.4.1--0.4.2.sql

@@ -1,2 +0,0 @@
--- complain if script is sourced in psql, rather than via CREATE EXTENSION
-\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.2'" to load this file. \quit

src/ivfbuild.c

@@ -438,8 +438,8 @@ ComputeCenters(IvfflatBuildState * buildstate)
 		{
 			ereport(NOTICE,
 					(errmsg("ivfflat index created with little data"),
-					 errdetail("This will cause low recall."),
+					 errdetail("this will cause poor recall"),
-					 errhint("Drop the index until the table has more data.")));
+					 errhint("drop the index until the table has more data")));
 		}
 	}
 

src/ivfflat.h

@@ -206,6 +206,7 @@ typedef struct IvfflatScanOpaqueData
 
 	/* Lists */
 	pairingheap *listQueue;
+	double		minDistance;
 	IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER];	/* must come last */
 }			IvfflatScanOpaqueData;
 

src/ivfscan.c

@@ -60,6 +60,9 @@ GetScanLists(IndexScanDesc scan, Datum value)
 			/* Use procinfo from the index instead of scan key for performance */
 			distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
 
+			if (distance < so->minDistance)
+				so->minDistance = distance;
+
 			if (listCount < so->probes)
 			{
 				scanlist = &so->lists[listCount];
@@ -129,7 +132,13 @@ GetScanItems(IndexScanDesc scan, Datum value)
 	/* Search closest probes lists */
 	while (!pairingheap_is_empty(so->listQueue))
 	{
-		searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
+		IvfflatScanList *scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
+
+		/* Query-aware dynamic pruning */
+		if (fabs(scanlist->distance) > 1.5 * fabs(so->minDistance))
+			continue;
+
+		searchPage = scanlist->startPage;
 
 		/* Search all entry pages for list */
 		while (BlockNumberIsValid(searchPage))
@@ -174,8 +183,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
 	if (tuples < 100)
 		ereport(DEBUG1,
 				(errmsg("index scan found few tuples"),
-				 errdetail("Index may have been created with little data."),
+				 errdetail("index may have been created without data or lists is too high"),
-				 errhint("Recreate the index and possibly decrease lists.")));
+				 errhint("recreate the index and possibly decrease lists")));
 
 	tuplesort_performsort(so->sortstate);
 }
@@ -252,6 +261,7 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
 
 	so->first = true;
 	pairingheap_reset(so->listQueue);
+	so->minDistance = DBL_MAX;
 
 	if (keys && scan->numberOfKeys > 0)
 		memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));

src/vector.c

@@ -396,8 +396,10 @@ array_to_vector(PG_FUNCTION_ARGS)
 	get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
 	deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp);
 
-	CheckDim(nelemsp);
+	if (typmod == -1)
-	CheckExpectedDim(typmod, nelemsp);
+		CheckDim(nelemsp);
+	else
+		CheckExpectedDim(typmod, nelemsp);
 
 	result = InitVector(nelemsp);
 	for (i = 0; i < nelemsp; i++)
@@ -950,7 +952,6 @@ vector_avg(PG_FUNCTION_ARGS)
 
 	/* Create vector */
 	dim = STATE_DIMS(statearray);
-	CheckDim(dim);
 	result = InitVector(dim);
 	for (int i = 0; i < dim; i++)
 	{

test/expected/cast.out

@@ -46,8 +46,6 @@ SELECT '[1,2,3]'::vector::real[];
 
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
 ERROR:  vector cannot have more than 16000 dimensions
-SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
-ERROR:  vector cannot have more than 16000 dimensions
 -- ensure no error
 SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
  ?column? 

test/expected/functions.out

@@ -102,5 +102,3 @@ SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
 
 SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
 ERROR:  expected 2 dimensions, not 1
-SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
-ERROR:  vector cannot have more than 16000 dimensions

test/sql/cast.sql

@@ -10,7 +10,6 @@ SELECT '{-Infinity}'::real[]::vector;
 SELECT '{}'::real[]::vector;
 SELECT '[1,2,3]'::vector::real[];
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
-SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
 
 -- ensure no error
 SELECT ARRAY[1,2,3] = ARRAY[1,2,3];

test/sql/functions.sql

@@ -24,4 +24,3 @@ SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
 SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
 SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
 SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
-SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat access method'
-default_version = '0.4.2'
+default_version = '0.4.1'
 module_pathname = '$libdir/vector'
 relocatable = true