Version bump to 0.4.3 [skip ci]

The compiler was complaining about a missing include due to the
addition of NaN/inf checks in 482a5f8b. Adding the include
silences the warnings.

.github/workflows/build.yml

@@ -8,6 +8,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - postgres: 16
+            os: ubuntu-22.04
           - postgres: 15
             os: ubuntu-22.04
           - postgres: 14
@@ -25,6 +27,8 @@ jobs:
           postgres-version: ${{ matrix.postgres }}
           dev-files: true
       - run: make
+        env:
+          PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
       - run: |
           export PG_CONFIG=`which pg_config`
           sudo --preserve-env=PG_CONFIG make install
@@ -44,6 +48,8 @@ jobs:
         with:
           postgres-version: 14
       - run: make
+        env:
+          PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter
       - run: make install
       - run: make installcheck
       - if: ${{ failure() }}
@@ -70,3 +76,22 @@ jobs:
           nmake /NOLOGO /F Makefile.win clean && ^
           nmake /NOLOGO /F Makefile.win uninstall
         shell: cmd
+  i386:
+    runs-on: ubuntu-latest
+    container:
+      image: debian:11
+      options: --platform linux/386
+    steps:
+      - run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-13 postgresql-server-dev-13 sudo
+      - run: service postgresql start
+      - run: |
+          git clone https://github.com/${{ github.repository }}.git pgvector
+          cd pgvector
+          git checkout ${{ github.ref }}
+          make
+          make install
+          chown -R postgres .
+          sudo -u postgres make installcheck
+          sudo -u postgres make prove_installcheck
+        env:
+          PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare

CHANGELOG.md

@@ -1,6 +1,18 @@
-## 0.4.2 (unreleased)
+## 0.4.3 (2023-06-10)
+
+- Improved cost estimation
+- Improved support for spaces with text representation
+- Fixed infinite and NaN values with binary representation
+- Fixed infinite values with vector addition and subtraction
+- Fixed infinite values with list centers
+- Fixed compilation error when `float8` is pass by reference
+- Fixed compilation error on PowerPC
+- Fixed segmentation fault with index creation on i386
+
+## 0.4.2 (2023-05-13)
 
 - Added notice when index created with little data
+- Fixed dimensions check for some direct function calls
 - Fixed installation error with Postgres 12.0-12.2
 
 ## 0.4.1 (2023-03-21)

LICENSE

@@ -1,4 +1,4 @@
-Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
 
 Portions Copyright (c) 1994, The Regents of the University of California
 

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.4.1",
+	"version": "0.4.3",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.4.1",
+			"version": "0.4.3",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.4.1
+EXTVERSION = 0.4.3
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)
@@ -19,6 +19,11 @@ ifeq ($(shell uname -s), Darwin)
 	endif
 endif
 
+# PowerPC doesn't support -march=native
+ifneq ($(filter ppc64%, $(shell uname -m)), )
+	OPTFLAGS =
+endif
+
 # For auto-vectorization:
 # - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
 # - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -63,3 +68,9 @@ dist:
 
 docker:
 	docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest .
+
+.PHONY: docker-release
+
+docker-release:
+	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 -t ankane/pgvector:latest .
+	docker buildx build --push --platform linux/amd64,linux/arm64 -t ankane/pgvector:v$(EXTVERSION) .

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.4.1
+EXTVERSION = 0.4.3
 
 OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
 

README.md

@@ -8,6 +8,8 @@ Supports
 - L2 distance, inner product, and cosine distance
 - any [language](#languages) with a Postgres client
 
+Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
+
 [![Build Status](https://github.com/pgvector/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector/actions)
 
 ## Installation
@@ -16,24 +18,24 @@ Compile and install the extension (supports Postgres 11+)
 
 ```sh
 cd /tmp
-git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.3 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
 ```
 
-Then load it in databases where you want to use it
-
-```sql
-CREATE EXTENSION vector;
-```
-
 See the [installation notes](#installation-notes) if you run into issues
 
-You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [Yum](#yum), or [conda-forge](#conda-forge)
+You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres)
 
 ## Getting Started
 
+Enable the extension (do this once in each database where you want to use it)
+
+```tsql
+CREATE EXTENSION vector;
+```
+
 Create a vector column with 3 dimensions
 
 ```sql
@@ -127,7 +129,7 @@ SELECT embedding <-> '[3,1,2]' AS distance FROM items;
 
 For inner product, multiply by -1 (since `<#>` returns the negative inner product)
 
-```sql
+```tsql
 SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items;
 ```
 
@@ -221,26 +223,42 @@ The phases are:
 
 Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
 
-### Partial Indexes
+### Filtering
 
-Consider [partial indexes](https://www.postgresql.org/docs/current/indexes-partial.html) for queries with a `WHERE` clause
+There are a few ways to index nearest neighbor queries with a `WHERE` clause
 
 ```sql
 SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
 ```
 
-can be indexed with:
+Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
 
 ```sql
-CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) WHERE (category_id = 123);
+CREATE INDEX ON items (category_id);
 ```
 
-To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`.
+Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
+
+```sql
+CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)
+    WHERE (category_id = 123);
+```
+
+Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
 
 ```sql
 CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
 ```
 
+## Hybrid Search
+
+Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search ([Python example](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py)).
+
+```sql
+SELECT id, content FROM items, to_tsquery('hello & search') query
+    WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
+```
+
 ## Performance
 
 Use `EXPLAIN ANALYZE` to debug performance.
@@ -259,7 +277,7 @@ SET max_parallel_workers_per_gather = 4;
 
 If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance.
 
-```sql
+```tsql
 SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
 ```
 
@@ -307,10 +325,7 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
 
 #### What if I want to index vectors with more than 2,000 dimensions?
 
-Two things you can try are:
+You’ll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
-
-1. use dimensionality reduction
-2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
 
 #### Why am I seeing less results after adding an index?
 
@@ -358,7 +373,11 @@ If your machine has multiple Postgres installations, specify the path to [pg_con
 export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
 ```
 
-Then re-run the installation instructions (run `make clean` before `make` if needed)
+Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
+
+```sh
+sudo --preserve-env=PG_CONFIG make install
+```
 
 ### Missing Header
 
@@ -367,7 +386,7 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
 For Ubuntu and Debian, use:
 
 ```sh
-sudo apt-get install postgresql-server-dev-15
+sudo apt install postgresql-server-dev-15
 ```
 
 Note: Replace `15` with your Postgres server version
@@ -378,7 +397,7 @@ Support for Windows is currently experimental. Use `nmake` to build:
 
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\15"
-git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.3 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -399,9 +418,9 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.3 https://github.com/pgvector/pgvector.git
 cd pgvector
-docker build -t pgvector .
+docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
 ```
 
 ### Homebrew
@@ -412,6 +431,8 @@ With Homebrew Postgres, you can use:
 brew install pgvector
 ```
 
+Note: This only adds it to the `postgresql@14` formula
+
 ### PGXN
 
 Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) with:
@@ -420,6 +441,16 @@ Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) wi
 pgxn install vector
 ```
 
+### APT
+
+Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
+
+```sh
+sudo apt install postgresql-15-pgvector
+```
+
+Note: Replace `15` with your Postgres server version
+
 ### Yum
 
 RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
@@ -442,16 +473,18 @@ conda install -c conda-forge pgvector
 
 This method is [community-maintained](https://github.com/conda-forge/pgvector-feedstock) by [@mmcauliffe](https://github.com/mmcauliffe)
 
+### Postgres.app
+
+Download the [latest release](https://postgresapp.com/downloads.html) with Postgres 15+.
+
 ## Hosted Postgres
 
 pgvector is available on [these providers](https://github.com/pgvector/pgvector/issues/54).
 
 To request a new extension on other providers:
 
-- Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
 - Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
-- Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
+- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/managed-database/p/pgvector-extension-for-postgresql)
-- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
 - Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
 
 ## Upgrading

sql/vector--0.4.1--0.4.2.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.2'" to load this file. \quit

sql/vector--0.4.2--0.4.3.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.3'" to load this file. \quit

src/ivfbuild.c

@@ -147,7 +147,7 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
 {
 	double		distance;
 	double		minDistance = DBL_MAX;
-	int			closestCenter = -1;
+	int			closestCenter = 0;
 	VectorArray centers = buildstate->centers;
 	TupleTableSlot *slot = buildstate->slot;
 	int			i;
@@ -438,8 +438,8 @@ ComputeCenters(IvfflatBuildState * buildstate)
 		{
 			ereport(NOTICE,
 					(errmsg("ivfflat index created with little data"),
-					 errdetail("this will cause poor recall"),
+					 errdetail("This will cause low recall."),
-					 errhint("drop the index until the table has more data")));
+					 errhint("Drop the index until the table has more data.")));
 		}
 	}
 
@@ -575,7 +575,7 @@ static void
 CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
 {
 	AttrNumber	attNums[] = {1};
-	Oid			sortOperators[] = {Float8LessOperator};
+	Oid			sortOperators[] = {Int4LessOperator};
 	Oid			sortCollations[] = {InvalidOid};
 	bool		nullsFirstFlags[] = {false};
 

src/ivfflat.c

@@ -7,6 +7,7 @@
 #include "ivfflat.h"
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
+#include "utils/spccache.h"
 
 #if PG_VERSION_NUM >= 120000
 #include "commands/progress.h"
@@ -63,13 +64,13 @@ ivfflatbuildphasename(int64 phasenum)
 static void
 ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 					Cost *indexStartupCost, Cost *indexTotalCost,
-					Selectivity *indexSelectivity, double *indexCorrelation
+					Selectivity *indexSelectivity, double *indexCorrelation,
-					,double *indexPages
+					double *indexPages)
-)
 {
 	GenericCosts costs;
 	int			lists;
 	double		ratio;
+	double		spc_seq_page_cost;
 	Relation	indexRel;
 #if PG_VERSION_NUM < 120000
 	List	   *qinfos;
@@ -88,6 +89,22 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 
 	MemSet(&costs, 0, sizeof(costs));
 
+	indexRel = index_open(path->indexinfo->indexoid, NoLock);
+	lists = IvfflatGetLists(indexRel);
+	index_close(indexRel, NoLock);
+
+	/* Get the ratio of lists that we need to visit */
+	ratio = ((double) ivfflat_probes) / lists;
+	if (ratio > 1.0)
+		ratio = 1.0;
+
+	/*
+	 * This gives us the subset of tuples to visit. This value is passed into
+	 * the generic cost estimator to determine the number of pages to visit
+	 * during the index scan.
+	 */
+	costs.numIndexTuples = path->indexinfo->tuples * ratio;
+
 #if PG_VERSION_NUM >= 120000
 	genericcostestimate(root, path, loop_count, &costs);
 #else
@@ -95,17 +112,31 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	genericcostestimate(root, path, loop_count, qinfos, &costs);
 #endif
 
-	indexRel = index_open(path->indexinfo->indexoid, NoLock);
+	get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
-	lists = IvfflatGetLists(indexRel);
-	index_close(indexRel, NoLock);
 
-	ratio = ((double) ivfflat_probes) / lists;
+	/* Adjust cost if needed since TOAST not included in seq scan cost */
-	if (ratio > 1)
+	if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
-		ratio = 1;
+	{
+		/* Change all page cost from random to sequential */
+		costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
 
-	costs.indexTotalCost *= ratio;
+		/* Remove cost of extra pages */
+		costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
+	}
+	else
+	{
+		/* Change some page cost from random to sequential */
+		costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+	}
 
-	/* Startup cost and total cost are same */
+	/*
+	 * If the list selectivity is lower than what is returned from the generic
+	 * cost estimator, use that.
+	 */
+	if (ratio < costs.indexSelectivity)
+		costs.indexSelectivity = ratio;
+
+	/* Use total cost since most work happens before first tuple is returned */
 	*indexStartupCost = costs.indexTotalCost;
 	*indexTotalCost = costs.indexTotalCost;
 	*indexSelectivity = costs.indexSelectivity;

src/ivfflat.h

@@ -206,7 +206,6 @@ typedef struct IvfflatScanOpaqueData
 
 	/* Lists */
 	pairingheap *listQueue;
-	double		minDistance;
 	IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER];	/* must come last */
 }			IvfflatScanOpaqueData;
 

src/ivfinsert.c

@@ -23,6 +23,10 @@ FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo *
 	OffsetNumber offno;
 	OffsetNumber maxoffno;
 
+	/* Avoid compiler warning */
+	listInfo->blkno = nextblkno;
+	listInfo->offno = FirstOffsetNumber;
+
 	procinfo = index_getprocinfo(rel, 1, IVFFLAT_DISTANCE_PROC);
 	collation = rel->rd_indcollation[0];
 
@@ -39,7 +43,7 @@ FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo *
 			list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno));
 			distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, values[0], PointerGetDatum(&list->center)));
 
-			if (distance < minDistance)
+			if (distance < minDistance || !BlockNumberIsValid(*insertPage))
 			{
 				*insertPage = list->insertPage;
 				listInfo->blkno = nextblkno;

src/ivfkmeans.c

@@ -1,6 +1,7 @@
 #include "postgres.h"
 
 #include <float.h>
+#include <math.h>
 
 #include "ivfflat.h"
 #include "miscadmin.h"
@@ -211,7 +212,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 
 	/* Check memory requirements */
 	/* Add one to error message to ceil */
-	if (totalSize / 1024 > maintenance_work_mem)
+	if (totalSize > (Size) maintenance_work_mem * 1024L)
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
@@ -251,7 +252,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 	for (j = 0; j < numSamples; j++)
 	{
 		minDistance = DBL_MAX;
-		closestCenter = -1;
+		closestCenter = 0;
 
 		/* Find closest center */
 		for (k = 0; k < numCenters; k++)
@@ -398,6 +399,14 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
 
 			if (centerCounts[j] > 0)
 			{
+				/* Double avoids overflow, but requires more memory */
+				/* TODO Update bounds */
+				for (k = 0; k < dimensions; k++)
+				{
+					if (isinf(vec->x[k]))
+						vec->x[k] = vec->x[k] > 0 ? FLT_MAX : -FLT_MAX;
+				}
+
 				for (k = 0; k < dimensions; k++)
 					vec->x[k] /= centerCounts[j];
 			}
@@ -461,12 +470,29 @@ CheckCenters(Relation index, VectorArray centers)
 {
 	FmgrInfo   *normprocinfo;
 	Oid			collation;
+	Vector	   *vec;
 	int			i;
+	int			j;
 	double		norm;
 
 	if (centers->length != centers->maxlen)
 		elog(ERROR, "Not enough centers. Please report a bug.");
 
+	/* Ensure no NaN or infinite values */
+	for (i = 0; i < centers->length; i++)
+	{
+		vec = VectorArrayGet(centers, i);
+
+		for (j = 0; j < vec->dim; j++)
+		{
+			if (isnan(vec->x[j]))
+				elog(ERROR, "NaN detected. Please report a bug.");
+
+			if (isinf(vec->x[j]))
+				elog(ERROR, "Infinite value detected. Please report a bug.");
+		}
+	}
+
 	/* Ensure no duplicate centers */
 	/* Fine to sort in-place */
 	qsort(centers->items, centers->length, VECTOR_SIZE(centers->dim), CompareVectors);

src/ivfscan.c

@@ -60,9 +60,6 @@ GetScanLists(IndexScanDesc scan, Datum value)
 			/* Use procinfo from the index instead of scan key for performance */
 			distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
 
-			if (distance < so->minDistance)
-				so->minDistance = distance;
-
 			if (listCount < so->probes)
 			{
 				scanlist = &so->lists[listCount];
@@ -132,13 +129,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
 	/* Search closest probes lists */
 	while (!pairingheap_is_empty(so->listQueue))
 	{
-		IvfflatScanList *scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
+		searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
-
-		/* Query-aware dynamic pruning */
-		if (scanlist->distance > 2 * so->minDistance)
-			continue;
-
-		searchPage = scanlist->startPage;
 
 		/* Search all entry pages for list */
 		while (BlockNumberIsValid(searchPage))
@@ -179,12 +170,14 @@ GetScanItems(IndexScanDesc scan, Datum value)
 		}
 	}
 
+	FreeAccessStrategy(bas);
+
 	/* TODO Scan more lists */
 	if (tuples < 100)
 		ereport(DEBUG1,
 				(errmsg("index scan found few tuples"),
-				 errdetail("index may have been created without data or lists is too high"),
+				 errdetail("Index may have been created with little data."),
-				 errhint("recreate the index and possibly decrease lists")));
+				 errhint("Recreate the index and possibly decrease lists.")));
 
 	tuplesort_performsort(so->sortstate);
 }
@@ -261,7 +254,6 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
 
 	so->first = true;
 	pairingheap_reset(so->listQueue);
-	so->minDistance = DBL_MAX;
 
 	if (keys && scan->numberOfKeys > 0)
 		memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));

src/ivfvacuum.c

@@ -132,6 +132,8 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
 		}
 	}
 
+	FreeAccessStrategy(bas);
+
 	return stats;
 }
 

src/vector.c

@@ -72,18 +72,43 @@ CheckDim(int dim)
  * Ensure finite elements
  */
 static inline void
-CheckElement(float value)
+CheckElementOrFree(float value, Vector * vec)
 {
 	if (isnan(value))
+	{
+		if (vec != NULL)
+			pfree(vec);
+
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_EXCEPTION),
 				 errmsg("NaN not allowed in vector")));
-
+	}
 
 	if (isinf(value))
+	{
+		if (vec != NULL)
+			pfree(vec);
+
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_EXCEPTION),
 				 errmsg("infinite value not allowed in vector")));
+	}
+}
+
+/*
+ * Check for whitespace, since array_isspace() is static
+ */
+static inline bool
+vector_isspace(char ch)
+{
+	if (ch == ' ' ||
+		ch == '\t' ||
+		ch == '\n' ||
+		ch == '\r' ||
+		ch == '\v' ||
+		ch == '\f')
+		return true;
+	return false;
 }
 
 /*
@@ -150,6 +175,9 @@ vector_in(PG_FUNCTION_ARGS)
 	char	   *stringEnd;
 	Vector	   *result;
 
+	while (vector_isspace(*str))
+		str++;
+
 	if (*str != '[')
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
@@ -169,7 +197,7 @@ vector_in(PG_FUNCTION_ARGS)
 
 		/* Use strtof like float4in to avoid a double-rounding problem */
 		x[dim] = strtof(pt, &stringEnd);
-		CheckElement(x[dim]);
+		CheckElementOrFree(x[dim], NULL);
 		dim++;
 
 		if (stringEnd == pt)
@@ -177,6 +205,9 @@ vector_in(PG_FUNCTION_ARGS)
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
 					 errmsg("invalid input syntax for type vector: \"%s\"", pt)));
 
+		while (vector_isspace(*stringEnd))
+			stringEnd++;
+
 		if (*stringEnd != '\0' && *stringEnd != ']')
 			ereport(ERROR,
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
@@ -191,7 +222,13 @@ vector_in(PG_FUNCTION_ARGS)
 				 errmsg("malformed vector literal"),
 				 errdetail("Unexpected end of input.")));
 
-	if (stringEnd[1] != '\0')
+	stringEnd++;
+
+	/* Only whitespace is allowed after the closing brace */
+	while (vector_isspace(*stringEnd))
+		stringEnd++;
+
+	if (*stringEnd != '\0')
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
 				 errmsg("malformed vector literal"),
@@ -330,7 +367,10 @@ vector_recv(PG_FUNCTION_ARGS)
 
 	result = InitVector(dim);
 	for (i = 0; i < dim; i++)
+	{
 		result->x[i] = pq_getmsgfloat4(buf);
+		CheckElementOrFree(result->x[i], result);
+	}
 
 	PG_RETURN_POINTER(result);
 }
@@ -396,10 +436,8 @@ array_to_vector(PG_FUNCTION_ARGS)
 	get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
 	deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp);
 
-	if (typmod == -1)
+	CheckDim(nelemsp);
-		CheckDim(nelemsp);
+	CheckExpectedDim(typmod, nelemsp);
-	else
-		CheckExpectedDim(typmod, nelemsp);
 
 	result = InitVector(nelemsp);
 	for (i = 0; i < nelemsp; i++)
@@ -409,6 +447,7 @@ array_to_vector(PG_FUNCTION_ARGS)
 					(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
 					 errmsg("array must not containing NULLs")));
 
+		/* TODO Move outside loop in 0.5.0 */
 		if (ARR_ELEMTYPE(array) == INT4OID)
 			result->x[i] = DatumGetInt32(elemsp[i]);
 		else if (ARR_ELEMTYPE(array) == FLOAT8OID)
@@ -422,7 +461,7 @@ array_to_vector(PG_FUNCTION_ARGS)
 					(errcode(ERRCODE_DATA_EXCEPTION),
 					 errmsg("unsupported array type")));
 
-		CheckElement(result->x[i]);
+		CheckElementOrFree(result->x[i], result);
 	}
 
 	PG_RETURN_POINTER(result);
@@ -436,17 +475,19 @@ Datum
 vector_to_float4(PG_FUNCTION_ARGS)
 {
 	Vector	   *vec = PG_GETARG_VECTOR_P(0);
-	Datum	   *d;
+	Datum	   *datums;
 	ArrayType  *result;
 	int			i;
 
-	d = (Datum *) palloc(sizeof(Datum) * vec->dim);
+	datums = (Datum *) palloc(sizeof(Datum) * vec->dim);
 
 	for (i = 0; i < vec->dim; i++)
-		d[i] = Float4GetDatum(vec->x[i]);
+		datums[i] = Float4GetDatum(vec->x[i]);
 
 	/* Use TYPALIGN_INT for float4 */
-	result = construct_array(d, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT);
+	result = construct_array(datums, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT);
+
+	pfree(datums);
 
 	PG_RETURN_POINTER(result);
 }
@@ -658,6 +699,16 @@ vector_add(PG_FUNCTION_ARGS)
 	for (int i = 0, imax = a->dim; i < imax; i++)
 		rx[i] = ax[i] + bx[i];
 
+	/* Check for overflow */
+	for (int i = 0, imax = a->dim; i < imax; i++)
+	{
+		if (isinf(rx[i]))
+		{
+			pfree(result);
+			float_overflow_error();
+		}
+	}
+
 	PG_RETURN_POINTER(result);
 }
 
@@ -684,6 +735,16 @@ vector_sub(PG_FUNCTION_ARGS)
 	for (int i = 0, imax = a->dim; i < imax; i++)
 		rx[i] = ax[i] - bx[i];
 
+	/* Check for overflow */
+	for (int i = 0, imax = a->dim; i < imax; i++)
+	{
+		if (isinf(rx[i]))
+		{
+			pfree(result);
+			float_overflow_error();
+		}
+	}
+
 	PG_RETURN_POINTER(result);
 }
 
@@ -829,12 +890,12 @@ vector_accum(PG_FUNCTION_ARGS)
 	n = statevalues[0] + 1.0;
 
 	statedatums = CreateStateDatums(dim);
-	statedatums[0] = Float8GetDatumFast(n);
+	statedatums[0] = Float8GetDatum(n);
 
 	if (newarr)
 	{
 		for (int i = 0; i < dim; i++)
-			statedatums[i + 1] = Float8GetDatumFast((double) x[i]);
+			statedatums[i + 1] = Float8GetDatum((double) x[i]);
 	}
 	else
 	{
@@ -842,10 +903,14 @@ vector_accum(PG_FUNCTION_ARGS)
 		{
 			double		v = statevalues[i + 1] + x[i];
 
+			/* Check for overflow */
 			if (isinf(v))
+			{
+				pfree(statedatums);
 				float_overflow_error();
+			}
 
-			statedatums[i + 1] = Float8GetDatumFast(v);
+			statedatums[i + 1] = Float8GetDatum(v);
 		}
 	}
 
@@ -890,7 +955,7 @@ vector_combine(PG_FUNCTION_ARGS)
 		dim = STATE_DIMS(statearray2);
 		statedatums = CreateStateDatums(dim);
 		for (int i = 1; i <= dim; i++)
-			statedatums[i] = Float8GetDatumFast(statevalues2[i]);
+			statedatums[i] = Float8GetDatum(statevalues2[i]);
 	}
 	else if (n2 == 0.0)
 	{
@@ -898,7 +963,7 @@ vector_combine(PG_FUNCTION_ARGS)
 		dim = STATE_DIMS(statearray1);
 		statedatums = CreateStateDatums(dim);
 		for (int i = 1; i <= dim; i++)
-			statedatums[i] = Float8GetDatumFast(statevalues1[i]);
+			statedatums[i] = Float8GetDatum(statevalues1[i]);
 	}
 	else
 	{
@@ -910,14 +975,18 @@ vector_combine(PG_FUNCTION_ARGS)
 		{
 			double		v = statevalues1[i] + statevalues2[i];
 
+			/* Check for overflow */
 			if (isinf(v))
+			{
+				pfree(statedatums);
 				float_overflow_error();
+			}
 
-			statedatums[i] = Float8GetDatumFast(v);
+			statedatums[i] = Float8GetDatum(v);
 		}
 	}
 
-	statedatums[0] = Float8GetDatumFast(n);
+	statedatums[0] = Float8GetDatum(n);
 
 	result = construct_array(statedatums, dim + 1,
 							 FLOAT8OID,
@@ -940,7 +1009,6 @@ vector_avg(PG_FUNCTION_ARGS)
 	float8		n;
 	uint16		dim;
 	Vector	   *result;
-	float		v;
 
 	/* Check array before using */
 	statevalues = CheckStateArray(statearray, "vector_avg");
@@ -952,12 +1020,12 @@ vector_avg(PG_FUNCTION_ARGS)
 
 	/* Create vector */
 	dim = STATE_DIMS(statearray);
+	CheckDim(dim);
 	result = InitVector(dim);
 	for (int i = 0; i < dim; i++)
 	{
-		v = statevalues[i + 1] / n;
+		result->x[i] = statevalues[i + 1] / n;
-		CheckElement(v);
+		CheckElementOrFree(result->x[i], result);
-		result->x[i] = v;
 	}
 
 	PG_RETURN_POINTER(result);

test/expected/cast.out

@@ -46,6 +46,8 @@ SELECT '[1,2,3]'::vector::real[];
 
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
 ERROR:  vector cannot have more than 16000 dimensions
+SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
+ERROR:  vector cannot have more than 16000 dimensions
 -- ensure no error
 SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
  ?column? 

test/expected/functions.out

@@ -4,12 +4,16 @@ SELECT '[1,2,3]'::vector + '[4,5,6]';
  [5,7,9]
 (1 row)
 
+SELECT '[3e38]'::vector + '[3e38]';
+ERROR:  value out of range: overflow
 SELECT '[1,2,3]'::vector - '[4,5,6]';
   ?column?  
 ------------
  [-3,-3,-3]
 (1 row)
 
+SELECT '[-3e38]'::vector - '[3e38]';
+ERROR:  value out of range: overflow
 SELECT vector_dims('[1,2,3]');
  vector_dims 
 -------------
@@ -102,3 +106,5 @@ SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
 
 SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
 ERROR:  expected 2 dimensions, not 1
+SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
+ERROR:  vector cannot have more than 16000 dimensions

test/expected/input.out

@@ -10,6 +10,12 @@ SELECT '[-1,2,3]'::vector;
  [-1,2,3]
 (1 row)
 
+SELECT ' [ 1,  2 ,    3  ] '::vector(3);
+ vector  
+---------
+ [1,2,3]
+(1 row)
+
 SELECT '[1.23456]'::vector;
   vector   
 -----------
@@ -32,6 +38,10 @@ SELECT '[-Infinity,1]'::vector;
 ERROR:  infinite value not allowed in vector
 LINE 1: SELECT '[-Infinity,1]'::vector;
                ^
+SELECT '[4e38,1]'::vector;
+ERROR:  infinite value not allowed in vector
+LINE 1: SELECT '[4e38,1]'::vector;
+               ^
 SELECT '[1,2,3'::vector;
 ERROR:  malformed vector literal
 LINE 1: SELECT '[1,2,3'::vector;

test/sql/cast.sql

@@ -10,6 +10,7 @@ SELECT '{-Infinity}'::real[]::vector;
 SELECT '{}'::real[]::vector;
 SELECT '[1,2,3]'::vector::real[];
 SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
+SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
 
 -- ensure no error
 SELECT ARRAY[1,2,3] = ARRAY[1,2,3];

test/sql/functions.sql

@@ -1,5 +1,7 @@
 SELECT '[1,2,3]'::vector + '[4,5,6]';
+SELECT '[3e38]'::vector + '[3e38]';
 SELECT '[1,2,3]'::vector - '[4,5,6]';
+SELECT '[-3e38]'::vector - '[3e38]';
 
 SELECT vector_dims('[1,2,3]');
 
@@ -24,3 +26,4 @@ SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
 SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
 SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
 SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
+SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;

test/sql/input.sql

@@ -1,10 +1,12 @@
 SELECT '[1,2,3]'::vector;
 SELECT '[-1,2,3]'::vector;
+SELECT ' [ 1,  2 ,    3  ] '::vector(3);
 SELECT '[1.23456]'::vector;
 SELECT '[hello,1]'::vector;
 SELECT '[NaN,1]'::vector;
 SELECT '[Infinity,1]'::vector;
 SELECT '[-Infinity,1]'::vector;
+SELECT '[4e38,1]'::vector;
 SELECT '[1,2,3'::vector;
 SELECT '[1,2,3]9'::vector;
 SELECT '1,2,3'::vector;

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat access method'
-default_version = '0.4.1'
+default_version = '0.4.3'
 module_pathname = '$libdir/vector'
 relocatable = true