Version bump to 0.2.3 [skip ci]

.github/workflows/build.yml

@@ -2,7 +2,6 @@ name: build
 on: [push, pull_request]
 jobs:
   build:
-    if: "!contains(github.event.head_commit.message, '[skip ci]')"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -34,6 +33,6 @@ jobs:
     - if: ${{ startsWith(matrix.os, 'macos') }}
       run: |
         brew install cpanm && cpanm IPC::Run
-        wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_13_2.tar.gz
+        wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_1.tar.gz
-        tar xf REL_13_2.tar.gz
+        tar xf REL_14_1.tar.gz
-        make prove_installcheck PROVE=prove PERL5LIB=postgres-REL_13_2/src/test/perl
+        make prove_installcheck PROVE=prove PERL5LIB=postgres-REL_14_1/src/test/perl

CHANGELOG.md

@@ -1,4 +1,13 @@
-## 0.2.1 (unreleased)
+## 0.2.3 (2022-01-30)
+
+- Added build progress for Postgres 12+
+- Improved interrupt handling during index creation
+
+## 0.2.2 (2022-01-15)
+
+- Fixed compilation error on Mac ARM
+
+## 0.2.1 (2022-01-02)
 
 - Fixed `operator is not unique` error
 

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.2.1",
+	"version": "0.2.3",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,18 +20,18 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.2.1",
+			"version": "0.2.3",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},
 	"resources": {
-		"homepage": "https://github.com/ankane/pgvector",
+		"homepage": "https://github.com/pgvector/pgvector",
 		"bugtracker": {
-			"web": "https://github.com/ankane/pgvector/issues"
+			"web": "https://github.com/pgvector/pgvector/issues"
 		},
 		"repository": {
-			"url":  "https://github.com/ankane/pgvector.git",
+			"url":  "https://github.com/pgvector/pgvector.git",
-			"web":  "https://github.com/ankane/pgvector",
+			"web":  "https://github.com/pgvector/pgvector",
 			"type": "git"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.2.1
+EXTVERSION = 0.2.3
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)
@@ -11,6 +11,13 @@ REGRESS_OPTS = --inputdir=test
 
 OPTFLAGS = -march=native
 
+# Mac ARM doesn't support -march=native
+ifeq ($(shell uname -s), Darwin)
+	ifeq ($(shell uname -p), arm)
+		OPTFLAGS =
+	endif
+endif
+
 # For auto-vectorization:
 # - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
 # - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html

README.md

@@ -10,14 +10,14 @@ SELECT * FROM table ORDER BY column <-> '[1,2,3]' LIMIT 5;
 
 Supports L2 distance, inner product, and cosine distance
 
-[![Build Status](https://github.com/ankane/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/ankane/pgvector/actions)
+[![Build Status](https://github.com/pgvector/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector/actions)
 
 ## Installation
 
 Compile and install the extension (supports Postgres 9.6+)
 
 ```sh
-git clone --branch v0.2.1 https://github.com/ankane/pgvector.git
+git clone --branch v0.2.3 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -120,6 +120,24 @@ SELECT ...
 COMMIT;
 ```
 
+### Indexing Progress [unreleased]
+
+Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
+
+```sql
+SELECT phase, tuples_done, tuples_total FROM pg_stat_progress_create_index;
+```
+
+The phases are:
+
+1. `initializing`
+2. `sampling table`
+3. `performing k-means`
+4. `sorting tuples`
+5. `loading tuples`
+
+Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
+
 ### Partial Indexes
 
 Consider [partial indexes](https://www.postgresql.org/docs/current/indexes-partial.html) for queries with a `WHERE` clause
@@ -174,18 +192,35 @@ vector_norm(vector) | Euclidean norm
 
 Libraries that use pgvector:
 
-- [pgvector-python](https://github.com/ankane/pgvector-python) (Python)
+- [pgvector-python](https://github.com/pgvector/pgvector-python) (Python)
 - [Neighbor](https://github.com/ankane/neighbor) (Ruby)
-- [pgvector-node](https://github.com/ankane/pgvector-node) (Node.js)
+- [pgvector-node](https://github.com/pgvector/pgvector-node) (Node.js)
-- [pgvector-go](https://github.com/ankane/pgvector-go) (Go)
+- [pgvector-go](https://github.com/pgvector/pgvector-go) (Go)
-- [pgvector-rust](https://github.com/ankane/pgvector-rust) (Rust)
+- [pgvector-rust](https://github.com/pgvector/pgvector-rust) (Rust)
-- [pgvector-cpp](https://github.com/ankane/pgvector-cpp) (C++)
+- [pgvector-cpp](https://github.com/pgvector/pgvector-cpp) (C++)
+
+## Frequently Asked Questions
+
+#### How many vectors can be stored in a single table?
+
+A non-partitioned table has a limit of 32 TB by default in Postgres. A partitioned table can have thousands of partitions of that size.
+
+#### Is replication supported?
+
+Yes, pgvector uses the write-ahead log (WAL), which allows for replication and point-in-time recovery.
+
+#### What if my data has more than 1024 dimensions?
+
+Two things you can try are:
+
+1. use dimensionality reduction
+2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/vector.h`
 
 ## Additional Installation Methods
 
 ### Docker
 
-Get the [Docker image](https://hub.docker.com/repository/docker/ankane/pgvector) with:
+Get the [Docker image](https://hub.docker.com/r/ankane/pgvector) with:
 
 ```sh
 docker pull ankane/pgvector
@@ -196,7 +231,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
 You can also build the image manually
 
 ```sh
-git clone --branch v0.2.1 https://github.com/ankane/pgvector.git
+git clone --branch v0.2.3 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build -t pgvector .
 ```
@@ -206,7 +241,7 @@ docker build -t pgvector .
 On Mac with Homebrew Postgres, you can use:
 
 ```sh
-brew install ankane/brew/pgvector
+brew install pgvector/brew/pgvector
 ```
 
 ### PGXN
@@ -223,7 +258,7 @@ Some Postgres providers only support specific extensions. To request a new exten
 
 - Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
 - Google Cloud SQL - follow the instructions on [this page](https://cloud.google.com/sql/docs/postgres/extensions#requesting-support-for-a-new-extension)
-- DigitalOcean Managed Databases - follow the instructions on [this page](https://docs.digitalocean.com/products/databases/postgresql/resources/supported-extensions/#supported-extensions)
+- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
 - Azure Database for PostgreSQL - follow the instructions on [this page](https://docs.microsoft.com/en-us/azure/postgresql/concepts-extensions#next-steps)
 
 ## Upgrading
@@ -246,21 +281,21 @@ Thanks to:
 
 ## History
 
-View the [changelog](https://github.com/ankane/pgvector/blob/master/CHANGELOG.md)
+View the [changelog](https://github.com/pgvector/pgvector/blob/master/CHANGELOG.md)
 
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:
 
-- [Report bugs](https://github.com/ankane/pgvector/issues)
+- [Report bugs](https://github.com/pgvector/pgvector/issues)
-- Fix bugs and [submit pull requests](https://github.com/ankane/pgvector/pulls)
+- Fix bugs and [submit pull requests](https://github.com/pgvector/pgvector/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
 To get started with development:
 
 ```sh
-git clone https://github.com/ankane/pgvector.git
+git clone https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install

sql/vector--0.2.1--0.2.2.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.2.2'" to load this file. \quit

sql/vector--0.2.2--0.2.3.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.2.3'" to load this file. \quit

src/ivfbuild.c

@@ -7,8 +7,19 @@
 #include "miscadmin.h"
 #include "storage/bufmgr.h"
 
+#if PG_VERSION_NUM >= 140000
+#include "utils/backend_progress.h"
+#elif PG_VERSION_NUM >= 120000
+#include "pgstat.h"
+#endif
+
 #if PG_VERSION_NUM >= 120000
 #include "access/tableam.h"
+#include "commands/progress.h"
+#else
+#define PROGRESS_CREATEIDX_SUBPHASE 0
+#define PROGRESS_CREATEIDX_TUPLES_TOTAL 0
+#define PROGRESS_CREATEIDX_TUPLES_DONE 0
 #endif
 
 #if PG_VERSION_NUM >= 110000
@@ -25,6 +36,17 @@
 #define CALLBACK_ITEM_POINTER HeapTuple hup
 #endif
 
+/*
+ * Update build phase progress
+ */
+static inline void
+UpdateProgress(int index, int64 val)
+{
+#if PG_VERSION_NUM >= 120000
+	pgstat_progress_update_param(index, val);
+#endif
+}
+
 /*
  * Callback for sampling
  */
@@ -82,6 +104,8 @@ SampleRows(IvfflatBuildState * buildstate)
 	int			targsamples = buildstate->samples->maxlen;
 	BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
 
+	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_SAMPLE);
+
 	buildstate->rowstoskip = -1;
 
 	BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, random());
@@ -165,6 +189,8 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
 	 * need not save it.
 	 */
 	tuplesort_puttupleslot(buildstate->sortstate, slot);
+
+	buildstate->indtuples++;
 }
 
 /*
@@ -212,6 +238,7 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 	BlockNumber insertPage = InvalidBlockNumber;
 	Size		itemsz;
 	int			i;
+	int64		inserted = 0;
 
 #if PG_VERSION_NUM >= 120000
 	TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsMinimalTuple);
@@ -220,10 +247,18 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 #endif
 	TupleDesc	tupdesc = RelationGetDescr(index);
 
+	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
+
+	UpdateProgress(PROGRESS_CREATEIDX_TUPLES_TOTAL, buildstate->indtuples);
+
 	GetNextTuple(buildstate->sortstate, tupdesc, slot, &itup, &list);
 
 	for (i = 0; i < buildstate->centers->length; i++)
 	{
+		/* Can take a while, so ensure we can interrupt */
+		/* Needs to be called when no buffer locks are held */
+		CHECK_FOR_INTERRUPTS();
+
 		buf = IvfflatNewBuffer(index, forkNum);
 		IvfflatInitPage(index, &buf, &page, &state);
 
@@ -243,7 +278,7 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
 
 			pfree(itup);
 
-			buildstate->indtuples += 1;
+			UpdateProgress(PROGRESS_CREATEIDX_TUPLES_DONE, ++inserted);
 
 			GetNextTuple(buildstate->sortstate, tupdesc, slot, &itup, &list);
 		}
@@ -283,6 +318,12 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
 	buildstate->kmeansnormprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
 	buildstate->collation = index->rd_indcollation[0];
 
+	/* Require more than one dimension for spherical k-means */
+	/* Lists check for backwards compatibility */
+	/* TODO Remove lists check in 0.3.0 */
+	if (buildstate->kmeansnormprocinfo != NULL && buildstate->dimensions == 1 && buildstate->lists > 1)
+		elog(ERROR, "dimensions must be greater than one for this opclass");
+
 	/* Create tuple description for sorting */
 #if PG_VERSION_NUM >= 120000
 	buildstate->tupdesc = CreateTemplateTupleDesc(4);
@@ -342,6 +383,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
 		SampleRows(buildstate);
 
 	/* Calculate centers */
+	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
 	IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers);
 
 	/* Free samples before we allocate more memory */
@@ -432,6 +474,8 @@ CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
 	Oid			sortCollations[] = {InvalidOid};
 	bool		nullsFirstFlags[] = {false};
 
+	UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_SORT);
+
 #if PG_VERSION_NUM >= 110000
 	buildstate->sortstate = tuplesort_begin_heap(buildstate->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, maintenance_work_mem, NULL, false);
 #else

src/ivfflat.c

@@ -8,6 +8,10 @@
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
 
+#if PG_VERSION_NUM >= 120000
+#include "commands/progress.h"
+#endif
+
 int			ivfflat_probes;
 static relopt_kind ivfflat_relopt_kind;
 
@@ -30,6 +34,31 @@ _PG_init(void)
 							1, 1, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
 }
 
+/*
+ * Get the name of index build phase
+ */
+#if PG_VERSION_NUM >= 120000
+static char *
+ivfflatbuildphasename(int64 phasenum)
+{
+	switch (phasenum)
+	{
+		case PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE:
+			return "initializing";
+		case PROGRESS_IVFFLAT_PHASE_SAMPLE:
+			return "sampling table";
+		case PROGRESS_IVFFLAT_PHASE_KMEANS:
+			return "performing k-means";
+		case PROGRESS_IVFFLAT_PHASE_SORT:
+			return "sorting tuples";
+		case PROGRESS_IVFFLAT_PHASE_LOAD:
+			return "loading tuples";
+		default:
+			return NULL;
+	}
+}
+#endif
+
 /*
  * Estimate the cost of an index scan
  */
@@ -174,7 +203,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
 	amroutine->amoptions = ivfflatoptions;
 	amroutine->amproperty = NULL;	/* TODO AMPROP_DISTANCE_ORDERABLE */
 #if PG_VERSION_NUM >= 120000
-	amroutine->ambuildphasename = NULL;
+	amroutine->ambuildphasename = ivfflatbuildphasename;
 #endif
 	amroutine->amvalidate = ivfflatvalidate;
 	amroutine->ambeginscan = ivfflatbeginscan;

src/ivfflat.h

@@ -27,6 +27,13 @@
 #define IVFFLAT_DEFAULT_LISTS	100
 #define IVFFLAT_MAX_LISTS		32768
 
+/* Build phases */
+/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
+#define PROGRESS_IVFFLAT_PHASE_SAMPLE	2
+#define PROGRESS_IVFFLAT_PHASE_KMEANS	3
+#define PROGRESS_IVFFLAT_PHASE_SORT		4
+#define PROGRESS_IVFFLAT_PHASE_LOAD		5
+
 #define IVFFLAT_LIST_SIZE(_dim)	(offsetof(IvfflatListData, center) + VECTOR_SIZE(_dim))
 
 #define IvfflatPageGetOpaque(page)	((IvfflatPageOpaque) PageGetSpecialPointer(page))

test/expected/cast.out

@@ -42,3 +42,10 @@ SELECT '[1,2,3]'::vector::real[];
 
 SELECT array_agg(n)::vector FROM generate_series(1, 1025) n;
 ERROR:  vector cannot have more than 1024 dimensions
+-- ensure no error
+SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
+ ?column? 
+----------
+ t
+(1 row)
+

test/expected/functions.out

@@ -54,9 +54,3 @@ SELECT cosine_distance('[1,2]', '[0,0]');
 
 SELECT cosine_distance('[1,2]', '[3]');
 ERROR:  different vector dimensions 2 and 1
-SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
- ?column? 
-----------
- t
-(1 row)
-

test/sql/cast.sql

@@ -12,3 +12,6 @@ SELECT '{-Infinity}'::real[]::vector;
 SELECT '{}'::real[]::vector;
 SELECT '[1,2,3]'::vector::real[];
 SELECT array_agg(n)::vector FROM generate_series(1, 1025) n;
+
+-- ensure no error
+SELECT ARRAY[1,2,3] = ARRAY[1,2,3];

test/sql/functions.sql

@@ -16,5 +16,3 @@ SELECT inner_product('[1,2]', '[3]');
 SELECT round(cosine_distance('[1,2]', '[2,4]')::numeric, 5);
 SELECT cosine_distance('[1,2]', '[0,0]');
 SELECT cosine_distance('[1,2]', '[3]');
-
-SELECT ARRAY[1,2,3] = ARRAY[1,2,3];

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat access method'
-default_version = '0.2.1'
+default_version = '0.2.3'
 module_pathname = '$libdir/vector'
 relocatable = true