Adjust index scan cost based on selectivity [skip ci]

CHANGELOG.md

@@ -1,7 +1,6 @@
 ## 0.8.0 (unreleased)
 
 - Added casts for arrays to `sparsevec`
-- Improved cost estimation
 - Reduced memory usage for HNSW index scans
 - Dropped support for Postgres 12
 

Dockerfile

@@ -1,4 +1,4 @@
-ARG PG_MAJOR=17
+ARG PG_MAJOR=16
 FROM postgres:$PG_MAJOR
 ARG PG_MAJOR
 

Makefile

@@ -66,7 +66,7 @@ dist:
 	git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master
 
 # for Docker
-PG_MAJOR ?= 17
+PG_MAJOR ?= 16
 
 .PHONY: docker
 

README.md

@@ -52,8 +52,6 @@ nmake /F Makefile.win
 nmake /F Makefile.win install
 ```
 
-Note: Postgres 17 is not supported yet due to an upstream issue
-
 See the [installation notes](#installation-notes---windows) if you run into issues
 
 You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -102,8 +100,6 @@ Or add a vector column to an existing table
 ALTER TABLE items ADD COLUMN embedding vector(3);
 ```
 
-Also supports [half-precision](#half-precision-vectors), [binary](#binary-vectors), and [sparse](#sparse-vectors) vectors
-
 Insert vectors
 
 ```sql
@@ -149,8 +145,6 @@ Supported distance functions are:
 - `<#>` - (negative) inner product
 - `<=>` - cosine distance
 - `<+>` - L1 distance (added in 0.7.0)
-- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
-- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
 
 Get the nearest neighbors to a row
 
@@ -989,7 +983,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
 If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
 
 ```sh
-export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
+export PG_CONFIG=/Library/PostgreSQL/16/bin/pg_config
 ```
 
 Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1000,11 +994,11 @@ sudo --preserve-env=PG_CONFIG make install
 
 A few common paths on Mac are:
 
-- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
-- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
-- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
+- EDB installer - `/Library/PostgreSQL/16/bin/pg_config`
+- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@16/bin/pg_config`
+- Homebrew (x86-64) - `/usr/local/opt/postgresql@16/bin/pg_config`
 
-Note: Replace `17` with your Postgres server version
+Note: Replace `16` with your Postgres server version
 
 ### Missing Header
 
@@ -1013,10 +1007,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
 For Ubuntu and Debian, use:
 
 ```sh
-sudo apt install postgresql-server-dev-17
+sudo apt install postgresql-server-dev-16
 ```
 
-Note: Replace `17` with your Postgres server version
+Note: Replace `16` with your Postgres server version
 
 ### Missing SDK
 
@@ -1049,17 +1043,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
 Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
 
 ```sh
-docker pull pgvector/pgvector:pg17
+docker pull pgvector/pgvector:pg16
 ```
 
-This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
+This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
 
 You can also build the image manually:
 
 ```sh
 git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
 cd pgvector
-docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
+docker build --pull --build-arg PG_MAJOR=16 -t myuser/pgvector .
 ```
 
 ### Homebrew
@@ -1070,7 +1064,7 @@ With Homebrew Postgres, you can use:
 brew install pgvector
 ```
 
-Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
+Note: This only adds it to the `postgresql@14` formula
 
 ### PGXN
 
@@ -1085,22 +1079,22 @@ pgxn install vector
 Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
 
 ```sh
-sudo apt install postgresql-17-pgvector
+sudo apt install postgresql-16-pgvector
 ```
 
-Note: Replace `17` with your Postgres server version
+Note: Replace `16` with your Postgres server version
 
 ### Yum
 
 RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
 
 ```sh
-sudo yum install pgvector_17
+sudo yum install pgvector_16
 # or
-sudo dnf install pgvector_17
+sudo dnf install pgvector_16
 ```
 
-Note: Replace `17` with your Postgres server version
+Note: Replace `16` with your Postgres server version
 
 ### pkg
 

src/halfvec.c

@@ -159,6 +159,24 @@ CheckStateArray(ArrayType *statearray, const char *caller)
 	return (float8 *) ARR_DATA_PTR(statearray);
 }
 
+#if PG_VERSION_NUM < 120003
+static pg_noinline void
+float_overflow_error(void)
+{
+	ereport(ERROR,
+			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
+			 errmsg("value out of range: overflow")));
+}
+
+static pg_noinline void
+float_underflow_error(void)
+{
+	ereport(ERROR,
+			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
+			 errmsg("value out of range: underflow")));
+}
+#endif
+
 /*
  * Convert textual representation to internal representation
  */

src/hnsw.c

@@ -12,7 +12,6 @@
 #include "utils/float.h"
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
-#include "utils/spccache.h"
 
 #if PG_VERSION_NUM < 150000
 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -101,10 +100,6 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	GenericCosts costs;
 	int			m;
 	int			entryLevel;
-	int			layer0TuplesMax;
-	double		layer0Selectivity;
-	double		scalingFactor = 0.55;
-	double		spc_seq_page_cost;
 	Relation	index;
 
 	/* Never use index without order */
@@ -124,55 +119,15 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	HnswGetMetaPageInfo(index, &m, NULL);
 	index_close(index, NoLock);
 
-	/*
-	 * HNSW cost estimation follows a formula that accounts for the total
-	 * number of tuples indexed combined with the parameters that most
-	 * influence the duration of the index scan, namely: m - the number of
-	 * tuples that are scanned in each step of the HNSW graph traversal
-	 * ef_search - which influences the total number of steps taken at layer 0
-	 *
-	 * The source of the vector data can impact how many steps it takes to
-	 * converge on the set of vectors to return to the executor. Currently, we
-	 * use a hardcoded scaling factor (HNSWScanScalingFactor) to help
-	 * influence that, but this could later become a configurable parameter
-	 * based on the cost estimations.
-	 *
-	 * The tuple estimator formula is below:
-	 *
-	 * numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
-	 *
-	 * "entryLevel * m" represents the floor of tuples we need to scan to get
-	 * to layer 0 (L0).
-	 *
-	 * "layer0TuplesMax" is the estimated total number of tuples we'd scan at
-	 * L0 if we weren't discarding already visited tuples as part of the scan.
-	 *
-	 * "layer0Selectivity" estimates the percentage of tuples that are scanned
-	 * at L0, accounting for previously visited tuples, multiplied by the
-	 * "scalingFactor" (currently hardcoded).
-	 */
-	entryLevel = (int) (log(path->indexinfo->tuples + 1) * HnswGetMl(m));
-	layer0TuplesMax = HnswGetLayerM(m, 0) * hnsw_ef_search;
-	layer0Selectivity = (scalingFactor * log(path->indexinfo->tuples + 1)) /
-		(log(m) * (1 + log(hnsw_ef_search)));
+	/* Approximate entry level */
+	entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
 
-	costs.numIndexTuples = (entryLevel * m) +
-		(layer0TuplesMax * layer0Selectivity);
+	/* TODO Improve estimate of visited tuples (currently underestimates) */
+	/* Account for number of tuples (or entry level), m, and ef_search */
+	costs.numIndexTuples = (entryLevel + 2) * m;
 
 	genericcostestimate(root, path, loop_count, &costs);
 
-	get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
-
-	/* Adjust cost if needed since TOAST not included in seq scan cost */
-	if (costs.numIndexPages > path->indexinfo->rel->pages)
-	{
-		/* Change all page cost from random to sequential */
-		costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
-
-		/* Remove cost of extra pages */
-		costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
-	}
-
 	/* Use total cost since most work happens before first tuple is returned */
 	*indexStartupCost = costs.indexTotalCost;
 	*indexTotalCost = costs.indexTotalCost;

src/hnswutils.c

@@ -759,16 +759,9 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
 	page = BufferGetPage(buf);
 
 	ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
-
-	/* Ensure expected neighbors */
-	if (ntup->count != (element->level + 2) * m)
-	{
-		UnlockReleaseBuffer(buf);
-		return;
-	}
+	start = (element->level - lc) * m;
 
 	/* Copy to minimize lock time */
-	start = (element->level - lc) * m;
 	memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
 
 	UnlockReleaseBuffer(buf);

src/ivfflat.c

@@ -57,6 +57,39 @@ ivfflatbuildphasename(int64 phasenum)
 	}
 }
 
+/*
+ * Estimate the number of probes needed to get requested tuples
+ */
+static int
+EstimateProbes(PlannerInfo *root, IndexPath *path, int lists)
+{
+	double		selectivity = 1;
+	ListCell   *lc;
+	double		tuplesPerList;
+
+	/* Cannot estimate without limit */
+	/* limit_tuples includes offset */
+	if (root->limit_tuples < 0)
+		return 0;
+
+	/* Get the selectivity of non-index conditions */
+	foreach(lc, path->indexinfo->indrestrictinfo)
+	{
+		RestrictInfo *rinfo = lfirst(lc);
+
+		/* Skip DEFAULT_INEQ_SEL since it may be a distance filter */
+		if (rinfo->norm_selec >= 0 && rinfo->norm_selec <= 1 && rinfo->norm_selec != (Selectivity) DEFAULT_INEQ_SEL)
+			selectivity *= rinfo->norm_selec;
+	}
+
+	tuplesPerList = path->indexinfo->tuples * selectivity / (double) lists;
+	if (tuplesPerList == 0)
+		return lists;
+
+	/* No need to cap at this point */
+	return root->limit_tuples / tuplesPerList;
+}
+
 /*
  * Estimate the cost of an index scan
  */
@@ -68,6 +101,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 {
 	GenericCosts costs;
 	int			lists;
+	int			probes;
 	double		ratio;
 	double		spc_seq_page_cost;
 	Relation	index;
@@ -85,20 +119,25 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 
 	MemSet(&costs, 0, sizeof(costs));
 
-	genericcostestimate(root, path, loop_count, &costs);
-
 	index = index_open(path->indexinfo->indexoid, NoLock);
 	IvfflatGetMetaPageInfo(index, &lists, NULL);
 	index_close(index, NoLock);
 
+	probes = Max(ivfflat_probes, EstimateProbes(root, path, lists));
+
 	/* Get the ratio of lists that we need to visit */
-	ratio = ((double) ivfflat_probes) / lists;
+	ratio = ((double) probes) / lists;
 	if (ratio > 1.0)
 		ratio = 1.0;
 
-	/* Set startup cost since most work happens before first tuple is returned */
-	costs.indexStartupCost = costs.indexTotalCost * ratio;
-	costs.numIndexPages *= ratio;
+	/*
+	 * This gives us the subset of tuples to visit. This value is passed into
+	 * the generic cost estimator to determine the number of pages to visit
+	 * during the index scan.
+	 */
+	costs.numIndexTuples = path->indexinfo->tuples * ratio;
+
+	genericcostestimate(root, path, loop_count, &costs);
 
 	get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
 
@@ -106,25 +145,30 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
 	{
 		/* Change all page cost from random to sequential */
-		costs.indexStartupCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+		costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
 
 		/* Remove cost of extra pages */
-		costs.indexStartupCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
+		costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
 	}
 	else
 	{
 		/* Change some page cost from random to sequential */
-		costs.indexStartupCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
+		costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
 	}
 
-	*indexStartupCost = costs.indexStartupCost;
+	/*
+	 * If the list selectivity is lower than what is returned from the generic
+	 * cost estimator, use that.
+	 */
+	if (ratio < costs.indexSelectivity)
+		costs.indexSelectivity = ratio;
+
+	/* Use total cost since most work happens before first tuple is returned */
+	*indexStartupCost = costs.indexTotalCost;
 	*indexTotalCost = costs.indexTotalCost;
 	*indexSelectivity = costs.indexSelectivity;
 	*indexCorrelation = costs.indexCorrelation;
 	*indexPages = costs.numIndexPages;
-
-	Assert(*indexStartupCost > 0);
-	Assert(*indexTotalCost > 0);
 }
 
 /*

src/vector.c

@@ -155,6 +155,24 @@ CheckStateArray(ArrayType *statearray, const char *caller)
 	return (float8 *) ARR_DATA_PTR(statearray);
 }
 
+#if PG_VERSION_NUM < 120003
+static pg_noinline void
+float_overflow_error(void)
+{
+	ereport(ERROR,
+			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
+			 errmsg("value out of range: overflow")));
+}
+
+static pg_noinline void
+float_underflow_error(void)
+{
+	ereport(ERROR,
+			(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
+			 errmsg("value out of range: underflow")));
+}
+#endif
+
 /*
  * Convert textual representation to internal representation
  */

test/t/009_ivfflat_filtering.pl

@@ -5,7 +5,7 @@ use PostgreSQL::Test::Utils;
 use Test::More;
 
 my $dim = 3;
-my $nc = 50;
+my $nc = 100;
 my $limit = 20;
 
 my $array_sql = join(",", ('random()') x $dim);
@@ -94,15 +94,15 @@ like($explain, qr/Seq Scan/);
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query';
 ));
-like($explain, qr/Seq Scan/);
+# TODO Do not use index
+like($explain, qr/Index Scan using idx/);
 
 # Test attribute index
 $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# TODO Use attribute index
-like($explain, qr/Index Scan using idx/);
+like($explain, qr/Index Scan on attribute_idx/);
 
 # Test partial index
 $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5) WHERE (c = $c);");

test/t/017_hnsw_filtering.pl

@@ -18,13 +18,9 @@ $node->start;
 # Create table and index
 $node->safe_psql("postgres", "CREATE EXTENSION vector;");
 $node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);");
-$node->safe_psql("postgres", "CREATE TABLE cat (i int4 PRIMARY KEY, t text, b boolean);");
 $node->safe_psql("postgres",
 	"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;"
 );
-$node->safe_psql("postgres",
-	"INSERT INTO cat SELECT i, 'cat ' || i, i % 5 = 0 FROM generate_series(1, $nc) i;"
-);
 $node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
 $node->safe_psql("postgres", "ANALYZE tst;");
 
@@ -100,25 +96,13 @@ $explain = $node->safe_psql("postgres", qq(
 ));
 like($explain, qr/Seq Scan/);
 
-# Test join
-$explain = $node->safe_psql("postgres", qq(
-	EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c ORDER BY v <-> '$query' LIMIT $limit;
-));
-like($explain, qr/Index Scan using idx/);
-
-# Test join with attribute filtering
-$explain = $node->safe_psql("postgres", qq(
-	EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c WHERE cat.b = 't' ORDER BY v <-> '$query' LIMIT $limit;
-));
-like($explain, qr/Index Scan using idx/);
-
 # Test attribute index
 $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
 $explain = $node->safe_psql("postgres", qq(
 	EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
 ));
-# Use attribute index
-like($explain, qr/Bitmap Index Scan on attribute_idx/);
+# TODO Use attribute index
+like($explain, qr/Index Scan using idx/);
 
 # Test partial index
 $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);");

test/t/039_hnsw_cost.pl

@@ -1,51 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my @dims = (384, 1536);
-my $limit = 10;
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-
-for my $dim (@dims)
-{
-	my $array_sql = join(",", ('random()') x $dim);
-	my $n = $dim == 384 ? 3000 : 1000;
-
-	# Create table and index
-	$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
-	$node->safe_psql("postgres",
-		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, $n) i;"
-	);
-	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
-	$node->safe_psql("postgres", "ANALYZE tst;");
-
-	# Generate query
-	my @r = ();
-	for (1 .. $dim)
-	{
-		push(@r, rand());
-	}
-	my $query = "[" . join(",", @r) . "]";
-
-	my $explain = $node->safe_psql("postgres", qq(
-		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan using idx/);
-
-	$explain = $node->safe_psql("postgres", qq(
-		EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan using idx/);
-
-	$node->safe_psql("postgres", "DROP TABLE tst;");
-}
-
-done_testing();

test/t/040_ivfflat_cost.pl

@@ -1,50 +0,0 @@
-use strict;
-use warnings FATAL => 'all';
-use PostgreSQL::Test::Cluster;
-use PostgreSQL::Test::Utils;
-use Test::More;
-
-my @dims = (384, 1536);
-my $limit = 10;
-
-# Initialize node
-my $node = PostgreSQL::Test::Cluster->new('node');
-$node->init;
-$node->start;
-
-$node->safe_psql("postgres", "CREATE EXTENSION vector;");
-
-for my $dim (@dims)
-{
-	my $array_sql = join(",", ('random()') x $dim);
-
-	# Create table and index
-	$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
-	$node->safe_psql("postgres",
-		"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 6000) i;"
-	);
-	$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5);");
-	$node->safe_psql("postgres", "ANALYZE tst;");
-
-	# Generate query
-	my @r = ();
-	for (1 .. $dim)
-	{
-		push(@r, rand());
-	}
-	my $query = "[" . join(",", @r) . "]";
-
-	my $explain = $node->safe_psql("postgres", qq(
-		EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan using idx/);
-
-	$explain = $node->safe_psql("postgres", qq(
-		EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
-	));
-	like($explain, qr/Index Scan using idx/);
-
-	$node->safe_psql("postgres", "DROP TABLE tst;");
-}
-
-done_testing();