Version bump to 0.4.4 [skip ci]

.github/workflows/build.yml

@@ -60,6 +60,7 @@ jobs:
           wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
           tar xf REL_14_5.tar.gz
       - run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
+      - run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
   windows:
     runs-on: windows-latest
     if: ${{ !startsWith(github.ref_name, 'mac') }}

CHANGELOG.md

@@ -1,8 +1,14 @@
+## 0.4.4 (2023-06-12)
+
+- Improved error message for malformed vector literal
+- Fixed segmentation fault with text input
+- Fixed consecutive delimiters with text input
+
 ## 0.4.3 (2023-06-10)
 
 - Improved cost estimation
-- Improved support for spaces with text representation
-- Fixed infinite and NaN values with binary representation
+- Improved support for spaces with text input
+- Fixed infinite and NaN values with binary input
 - Fixed infinite values with vector addition and subtraction
 - Fixed infinite values with list centers
 - Fixed compilation error when `float8` is pass by reference

META.json

@@ -2,7 +2,7 @@
 	"name": "vector",
 	"abstract": "Open-source vector similarity search for Postgres",
 	"description": "Supports L2 distance, inner product, and cosine distance",
-	"version": "0.4.3",
+	"version": "0.4.4",
 	"maintainer": [
 		"Andrew Kane <andrew@ankane.org>"
 	],
@@ -20,7 +20,7 @@
 		"vector": {
 			"file": "sql/vector.sql",
 			"docfile": "README.md",
-			"version": "0.4.3",
+			"version": "0.4.4",
 			"abstract": "Open-source vector similarity search for Postgres"
 		}
 	},

Makefile

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.4.3
+EXTVERSION = 0.4.4
 
 MODULE_big = vector
 DATA = $(wildcard sql/*--*.sql)

Makefile.win

@@ -1,5 +1,5 @@
 EXTENSION = vector
-EXTVERSION = 0.4.3
+EXTVERSION = 0.4.4
 
 OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
 

README.md

@@ -18,7 +18,7 @@ Compile and install the extension (supports Postgres 11+)
 
 ```sh
 cd /tmp
-git clone --branch v0.4.3 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 make
 make install # may need sudo
@@ -163,7 +163,7 @@ Three keys to achieving good recall are:
 
 1. Create the index *after* the table has some data
 2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows
-3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `lists / 10` for up to 1M rows and `sqrt(lists)` for over 1M rows
+3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `sqrt(lists)`
 
 Add an index for each distance function you want to use.
 
@@ -397,7 +397,7 @@ Support for Windows is currently experimental. Use `nmake` to build:
 
 ```cmd
 set "PGROOT=C:\Program Files\PostgreSQL\15"
-git clone --branch v0.4.3 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 nmake /F Makefile.win
 nmake /F Makefile.win install
@@ -418,7 +418,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
 You can also build the image manually:
 
 ```sh
-git clone --branch v0.4.3 https://github.com/pgvector/pgvector.git
+git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
 cd pgvector
 docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
 ```

sql/vector--0.4.3--0.4.4.sql

@@ -0,0 +1,2 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.4'" to load this file. \quit

src/ivfbuild.c

@@ -568,6 +568,21 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
 }
 #endif
 
+/*
+ * Scan table for tuples to index
+ */
+static void
+ScanTable(IvfflatBuildState * buildstate)
+{
+#if PG_VERSION_NUM >= 120000
+	buildstate->reltuples = table_index_build_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
+												   true, true, BuildCallback, (void *) buildstate, NULL);
+#else
+	buildstate->reltuples = IndexBuildHeapScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
+											   true, BuildCallback, (void *) buildstate, NULL);
+#endif
+}
+
 /*
  * Create entry pages
  */
@@ -585,25 +600,17 @@ CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
 
 	/* Add tuples to sort */
 	if (buildstate->heap != NULL)
-	{
-#if PG_VERSION_NUM >= 120000
-		buildstate->reltuples = table_index_build_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
-													   true, true, BuildCallback, (void *) buildstate, NULL);
-#else
-		buildstate->reltuples = IndexBuildHeapScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
-												   true, BuildCallback, (void *) buildstate, NULL);
-#endif
-	}
+		IvfflatBench("assign tuples", ScanTable(buildstate));
 
 	/* Sort */
-	tuplesort_performsort(buildstate->sortstate);
+	IvfflatBench("sort tuples", tuplesort_performsort(buildstate->sortstate));
 
 #ifdef IVFFLAT_KMEANS_DEBUG
 	PrintKmeansMetrics(buildstate);
 #endif
 
 	/* Insert */
-	InsertTuples(buildstate->index, buildstate, forkNum);
+	IvfflatBench("load tuples", InsertTuples(buildstate->index, buildstate, forkNum));
 	tuplesort_end(buildstate->sortstate);
 }
 
@@ -621,7 +628,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
 	/* Create pages */
 	CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
 	CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
-	IvfflatBench("CreateEntryPages", CreateEntryPages(buildstate, forkNum));
+	CreateEntryPages(buildstate, forkNum);
 
 	FreeBuildState(buildstate);
 }

src/vector.c

@@ -42,7 +42,7 @@ CheckDims(Vector * a, Vector * b)
 }
 
 /*
- * Ensure expected dimension
+ * Ensure expected dimensions
  */
 static inline void
 CheckExpectedDim(int32 typmod, int dim)
@@ -53,7 +53,9 @@ CheckExpectedDim(int32 typmod, int dim)
 				 errmsg("expected %d dimensions, not %d", typmod, dim)));
 }
 
-
+/*
+ * Ensure valid dimensions
+ */
 static inline void
 CheckDim(int dim)
 {
@@ -72,27 +74,17 @@ CheckDim(int dim)
  * Ensure finite elements
  */
 static inline void
-CheckElementOrFree(float value, Vector * vec)
+CheckElement(float value)
 {
 	if (isnan(value))
-	{
-		if (vec != NULL)
-			pfree(vec);
-
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_EXCEPTION),
 				 errmsg("NaN not allowed in vector")));
-	}
 
 	if (isinf(value))
-	{
-		if (vec != NULL)
-			pfree(vec);
-
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_EXCEPTION),
 				 errmsg("infinite value not allowed in vector")));
-	}
 }
 
 /*
@@ -135,30 +127,6 @@ float_overflow_error(void)
 }
 #endif
 
-/*
- * Print vector - useful for debugging
- */
-void
-PrintVector(char *msg, Vector * vector)
-{
-	StringInfoData buf;
-	int			dim = vector->dim;
-	int			i;
-
-	initStringInfo(&buf);
-
-	appendStringInfoChar(&buf, '[');
-	for (i = 0; i < dim; i++)
-	{
-		if (i > 0)
-			appendStringInfoString(&buf, ",");
-		appendStringInfoString(&buf, float8out_internal(vector->x[i]));
-	}
-	appendStringInfoChar(&buf, ']');
-
-	elog(INFO, "%s = %s", msg, buf.data);
-}
-
 /*
  * Convert textual representation to internal representation
  */
@@ -174,6 +142,7 @@ vector_in(PG_FUNCTION_ARGS)
 	char	   *pt;
 	char	   *stringEnd;
 	Vector	   *result;
+	char	   *lit = pstrdup(str);
 
 	while (vector_isspace(*str))
 		str++;
@@ -181,7 +150,7 @@ vector_in(PG_FUNCTION_ARGS)
 	if (*str != '[')
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-				 errmsg("malformed vector literal: \"%s\"", str),
+				 errmsg("malformed vector literal: \"%s\"", lit),
 				 errdetail("Vector contents must start with \"[\".")));
 
 	str++;
@@ -195,15 +164,24 @@ vector_in(PG_FUNCTION_ARGS)
 					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 					 errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
 
+		while (vector_isspace(*pt))
+			pt++;
+
+		/* Check for empty string like float4in */
+		if (*pt == '\0')
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+					 errmsg("invalid input syntax for type vector: \"%s\"", lit)));
+
 		/* Use strtof like float4in to avoid a double-rounding problem */
 		x[dim] = strtof(pt, &stringEnd);
-		CheckElementOrFree(x[dim], NULL);
+		CheckElement(x[dim]);
 		dim++;
 
 		if (stringEnd == pt)
 			ereport(ERROR,
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-					 errmsg("invalid input syntax for type vector: \"%s\"", pt)));
+					 errmsg("invalid input syntax for type vector: \"%s\"", lit)));
 
 		while (vector_isspace(*stringEnd))
 			stringEnd++;
@@ -211,15 +189,15 @@ vector_in(PG_FUNCTION_ARGS)
 		if (*stringEnd != '\0' && *stringEnd != ']')
 			ereport(ERROR,
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-					 errmsg("invalid input syntax for type vector: \"%s\"", pt)));
+					 errmsg("invalid input syntax for type vector: \"%s\"", lit)));
 
 		pt = strtok(NULL, ",");
 	}
 
-	if (*stringEnd != ']')
+	if (stringEnd == NULL || *stringEnd != ']')
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-				 errmsg("malformed vector literal"),
+				 errmsg("malformed vector literal: \"%s\"", lit),
 				 errdetail("Unexpected end of input.")));
 
 	stringEnd++;
@@ -231,14 +209,25 @@ vector_in(PG_FUNCTION_ARGS)
 	if (*stringEnd != '\0')
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-				 errmsg("malformed vector literal"),
+				 errmsg("malformed vector literal: \"%s\"", lit),
 				 errdetail("Junk after closing right brace.")));
 
+	/* Ensure no consecutive delimiters since strtok skips */
+	for (pt = lit + 1; *pt != '\0'; pt++)
+	{
+		if (pt[-1] == ',' && *pt == ',')
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+					 errmsg("malformed vector literal: \"%s\"", lit)));
+	}
+
 	if (dim < 1)
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_EXCEPTION),
 				 errmsg("vector must have at least 1 dimension")));
 
+	pfree(lit);
+
 	CheckExpectedDim(typmod, dim);
 
 	result = InitVector(dim);
@@ -309,6 +298,18 @@ vector_out(PG_FUNCTION_ARGS)
 	PG_RETURN_CSTRING(buf);
 }
 
+/*
+ * Print vector - useful for debugging
+ */
+void
+PrintVector(char *msg, Vector * vector)
+{
+	char	   *out = DatumGetPointer(DirectFunctionCall1(vector_out, PointerGetDatum(vector)));
+
+	elog(INFO, "%s = %s", msg, out);
+	pfree(out);
+}
+
 /*
  * Convert type modifier
  */
@@ -369,7 +370,7 @@ vector_recv(PG_FUNCTION_ARGS)
 	for (i = 0; i < dim; i++)
 	{
 		result->x[i] = pq_getmsgfloat4(buf);
-		CheckElementOrFree(result->x[i], result);
+		CheckElement(result->x[i]);
 	}
 
 	PG_RETURN_POINTER(result);
@@ -461,7 +462,7 @@ array_to_vector(PG_FUNCTION_ARGS)
 					(errcode(ERRCODE_DATA_EXCEPTION),
 					 errmsg("unsupported array type")));
 
-		CheckElementOrFree(result->x[i], result);
+		CheckElement(result->x[i]);
 	}
 
 	PG_RETURN_POINTER(result);
@@ -703,10 +704,7 @@ vector_add(PG_FUNCTION_ARGS)
 	for (int i = 0, imax = a->dim; i < imax; i++)
 	{
 		if (isinf(rx[i]))
-		{
-			pfree(result);
 			float_overflow_error();
-		}
 	}
 
 	PG_RETURN_POINTER(result);
@@ -739,10 +737,7 @@ vector_sub(PG_FUNCTION_ARGS)
 	for (int i = 0, imax = a->dim; i < imax; i++)
 	{
 		if (isinf(rx[i]))
-		{
-			pfree(result);
 			float_overflow_error();
-		}
 	}
 
 	PG_RETURN_POINTER(result);
@@ -905,10 +900,7 @@ vector_accum(PG_FUNCTION_ARGS)
 
 			/* Check for overflow */
 			if (isinf(v))
-			{
-				pfree(statedatums);
 				float_overflow_error();
-			}
 
 			statedatums[i + 1] = Float8GetDatum(v);
 		}
@@ -977,10 +969,7 @@ vector_combine(PG_FUNCTION_ARGS)
 
 			/* Check for overflow */
 			if (isinf(v))
-			{
-				pfree(statedatums);
 				float_overflow_error();
-			}
 
 			statedatums[i] = Float8GetDatum(v);
 		}
@@ -1025,7 +1014,7 @@ vector_avg(PG_FUNCTION_ARGS)
 	for (int i = 0; i < dim; i++)
 	{
 		result->x[i] = statevalues[i + 1] / n;
-		CheckElementOrFree(result->x[i], result);
+		CheckElement(result->x[i]);
 	}
 
 	PG_RETURN_POINTER(result);

test/expected/input.out

@@ -4,13 +4,19 @@ SELECT '[1,2,3]'::vector;
  [1,2,3]
 (1 row)
 
-SELECT '[-1,2,3]'::vector;
-  vector  
-----------
- [-1,2,3]
+SELECT '[-1,-2,-3]'::vector;
+   vector   
+------------
+ [-1,-2,-3]
 (1 row)
 
-SELECT ' [ 1,  2 ,    3  ] '::vector(3);
+SELECT '[1.,2.,3.]'::vector;
+ vector  
+---------
+ [1,2,3]
+(1 row)
+
+SELECT ' [ 1,  2 ,    3  ] '::vector;
  vector  
 ---------
  [1,2,3]
@@ -23,7 +29,7 @@ SELECT '[1.23456]'::vector;
 (1 row)
 
 SELECT '[hello,1]'::vector;
-ERROR:  invalid input syntax for type vector: "hello"
+ERROR:  invalid input syntax for type vector: "[hello,1]"
 LINE 1: SELECT '[hello,1]'::vector;
                ^
 SELECT '[NaN,1]'::vector;
@@ -38,17 +44,35 @@ SELECT '[-Infinity,1]'::vector;
 ERROR:  infinite value not allowed in vector
 LINE 1: SELECT '[-Infinity,1]'::vector;
                ^
+SELECT '[1.5e38,-1.5e38]'::vector;
+       vector       
+--------------------
+ [1.5e+38,-1.5e+38]
+(1 row)
+
+SELECT '[1.5e+38,-1.5e+38]'::vector;
+       vector       
+--------------------
+ [1.5e+38,-1.5e+38]
+(1 row)
+
+SELECT '[1.5e-38,-1.5e-38]'::vector;
+       vector       
+--------------------
+ [1.5e-38,-1.5e-38]
+(1 row)
+
 SELECT '[4e38,1]'::vector;
 ERROR:  infinite value not allowed in vector
 LINE 1: SELECT '[4e38,1]'::vector;
                ^
 SELECT '[1,2,3'::vector;
-ERROR:  malformed vector literal
+ERROR:  malformed vector literal: "[1,2,3"
 LINE 1: SELECT '[1,2,3'::vector;
                ^
 DETAIL:  Unexpected end of input.
 SELECT '[1,2,3]9'::vector;
-ERROR:  malformed vector literal
+ERROR:  malformed vector literal: "[1,2,3]9"
 LINE 1: SELECT '[1,2,3]9'::vector;
                ^
 DETAIL:  Junk after closing right brace.
@@ -57,14 +81,36 @@ ERROR:  malformed vector literal: "1,2,3"
 LINE 1: SELECT '1,2,3'::vector;
                ^
 DETAIL:  Vector contents must start with "[".
+SELECT '['::vector;
+ERROR:  malformed vector literal: "["
+LINE 1: SELECT '['::vector;
+               ^
+DETAIL:  Unexpected end of input.
+SELECT '[,'::vector;
+ERROR:  malformed vector literal: "[,"
+LINE 1: SELECT '[,'::vector;
+               ^
+DETAIL:  Unexpected end of input.
 SELECT '[]'::vector;
 ERROR:  vector must have at least 1 dimension
 LINE 1: SELECT '[]'::vector;
                ^
 SELECT '[1,]'::vector;
-ERROR:  invalid input syntax for type vector: "]"
+ERROR:  invalid input syntax for type vector: "[1,]"
 LINE 1: SELECT '[1,]'::vector;
                ^
+SELECT '[1a]'::vector;
+ERROR:  invalid input syntax for type vector: "[1a]"
+LINE 1: SELECT '[1a]'::vector;
+               ^
+SELECT '[1,,3]'::vector;
+ERROR:  malformed vector literal: "[1,,3]"
+LINE 1: SELECT '[1,,3]'::vector;
+               ^
+SELECT '[1, ,3]'::vector;
+ERROR:  invalid input syntax for type vector: "[1, ,3]"
+LINE 1: SELECT '[1, ,3]'::vector;
+               ^
 SELECT '[1,2,3]'::vector(2);
 ERROR:  expected 2 dimensions, not 3
 SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);

test/sql/input.sql

@@ -1,17 +1,26 @@
 SELECT '[1,2,3]'::vector;
-SELECT '[-1,2,3]'::vector;
-SELECT ' [ 1,  2 ,    3  ] '::vector(3);
+SELECT '[-1,-2,-3]'::vector;
+SELECT '[1.,2.,3.]'::vector;
+SELECT ' [ 1,  2 ,    3  ] '::vector;
 SELECT '[1.23456]'::vector;
 SELECT '[hello,1]'::vector;
 SELECT '[NaN,1]'::vector;
 SELECT '[Infinity,1]'::vector;
 SELECT '[-Infinity,1]'::vector;
+SELECT '[1.5e38,-1.5e38]'::vector;
+SELECT '[1.5e+38,-1.5e+38]'::vector;
+SELECT '[1.5e-38,-1.5e-38]'::vector;
 SELECT '[4e38,1]'::vector;
 SELECT '[1,2,3'::vector;
 SELECT '[1,2,3]9'::vector;
 SELECT '1,2,3'::vector;
+SELECT '['::vector;
+SELECT '[,'::vector;
 SELECT '[]'::vector;
 SELECT '[1,]'::vector;
+SELECT '[1a]'::vector;
+SELECT '[1,,3]'::vector;
+SELECT '[1, ,3]'::vector;
 SELECT '[1,2,3]'::vector(2);
 
 SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);

vector.control

@@ -1,4 +1,4 @@
 comment = 'vector data type and ivfflat access method'
-default_version = '0.4.3'
+default_version = '0.4.4'
 module_pathname = '$libdir/vector'
 relocatable = true