Added search parameters to EXPLAIN [skip ci]

CHANGELOG.md

@@ -2,9 +2,9 @@
 
 - Added support for iterative index scans
 - Added casts for arrays to `sparsevec`
-- Improved cost estimation for better index selection when filtering
+- Improved cost estimation
-- Improved performance of HNSW index scans
 - Improved performance of HNSW inserts and on-disk index builds
+- Reduced memory usage for HNSW index scans
 - Dropped support for Postgres 12
 
 ## 0.7.4 (2024-08-05)

README.md

@@ -427,49 +427,25 @@ Note: `%` is only populated during the `loading tuples` phase
 
 ## Filtering
 
-There are a few ways to index nearest neighbor queries with a `WHERE` clause.
+There are a few ways to index nearest neighbor queries with a `WHERE` clause
 
 ```sql
 SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
 ```
 
-A good place to start is creating an index on the filter column. This can provide fast, exact nearest neighbor search in many cases. Postgres has a number of [index types](https://www.postgresql.org/docs/current/indexes-types.html) for this: B-tree (default), hash, GiST, SP-GiST, GIN, and BRIN.
+Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
 
 ```sql
 CREATE INDEX ON items (category_id);
 ```
 
-For multiple columns, consider a [multicolumn index](https://www.postgresql.org/docs/current/indexes-multicolumn.html).
+Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
-
-```sql
-CREATE INDEX ON items (location_id, category_id);
-```
-
-Exact indexes work well for conditions that match a low percentage of rows. Otherwise, [approximate indexes](#indexing) can work better.
-
-```sql
-CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
-```
-
-With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, increase `hnsw.ef_search`.
-
-```sql
-SET hnsw.ef_search = 200;
-```
-
-Starting with 0.8.0, you can enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
-
-```sql
-SET hnsw.iterative_scan = strict_order;
-```
-
-If filtering by only a few distinct values, consider [partial indexing](https://www.postgresql.org/docs/current/indexes-partial.html).
 
 ```sql
 CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WHERE (category_id = 123);
 ```
 
-If filtering by many different values, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html).
+Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
 
 ```sql
 CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
@@ -479,9 +455,9 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
 
 *Unreleased*
 
-With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
+With approximate indexes, queries with filtering can return less results (due to post-filtering). Starting with 0.8.0, you can enable iterative index scans. If too few results from the initial scan match the filters, the scan will resume until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). This can significantly improve recall.
 
-Iterative scans can use strict or relaxed ordering.
+There are two modes for iterative scans: strict and relaxed.
 
 Strict ensures results are in the exact order by distance
 
@@ -517,7 +493,7 @@ Note: Place any other filters inside the CTE
 
 ### Iterative Scan Options
 
-Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends.
+Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends
 
 #### HNSW
 
@@ -535,7 +511,18 @@ Specify the max amount of memory to use, as a multiple of `work_mem` (1 by defau
 SET hnsw.scan_mem_multiplier = 2;
 ```
 
-Note: Try increasing this if increasing `hnsw.max_scan_tuples` does not improve recall
+You can see when increasing this is needed by enabling debug messages
+
+```sql
+SET client_min_messages = debug1;
+```
+
+which will show when a scan reaches the memory limit
+
+```text
+DEBUG:  hnsw index scan reached memory limit after 20000 tuples
+HINT:  Increase hnsw.scan_mem_multiplier to scan more tuples.
+```
 
 #### IVFFlat
 

src/hnsw.c

@@ -77,21 +77,21 @@ HnswInit(void)
 
 	DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
 							"Valid range is 1..1000.", &hnsw_ef_search,
-							HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
+							HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	DefineCustomEnumVariable("hnsw.iterative_scan", "Sets the mode for iterative scans",
 							 NULL, &hnsw_iterative_scan,
-							 HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
+							 HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	/* This is approximate and does not affect the initial scan */
 	DefineCustomIntVariable("hnsw.max_scan_tuples", "Sets the max number of tuples to visit for iterative scans",
 							NULL, &hnsw_max_scan_tuples,
-							20000, 1, INT_MAX, PGC_USERSET, 0, NULL, NULL, NULL);
+							20000, 1, INT_MAX, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	/* Same range as hash_mem_multiplier */
 	DefineCustomRealVariable("hnsw.scan_mem_multiplier", "Sets the multiple of work_mem to use for iterative scans",
 							 NULL, &hnsw_scan_mem_multiplier,
-							 1, 1, 1000, PGC_USERSET, 0, NULL, NULL, NULL);
+							 1, 1, 1000, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	MarkGUCPrefixReserved("hnsw");
 }

src/hnswscan.c

@@ -240,8 +240,8 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 			if (so->discarded == NULL)
 				break;
 
-			/* Reached max number of tuples or memory limit */
+			/* Reached max number of tuples */
-			if (so->tuples >= hnsw_max_scan_tuples || MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
+			if (so->tuples >= hnsw_max_scan_tuples)
 			{
 				if (pairingheap_is_empty(so->discarded))
 					break;
@@ -249,6 +249,21 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
 				/* Return remaining tuples */
 				so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
 			}
+			/* Prevent scans from consuming too much memory */
+			else if (MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
+			{
+				if (pairingheap_is_empty(so->discarded))
+				{
+					ereport(DEBUG1,
+							(errmsg("hnsw index scan reached memory limit after " INT64_FORMAT " tuples", so->tuples),
+							 errhint("Increase hnsw.scan_mem_multiplier to scan more tuples.")));
+
+					break;
+				}
+
+				/* Return remaining tuples */
+				so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
+			}
 			else
 			{
 				/*

src/hnswutils.c

@@ -722,8 +722,6 @@ CountElement(HnswElement skipElement, HnswElement e)
 static void
 HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize)
 {
-	uint32		hashes[HNSW_MAX_M * 2];
-
 	/* Get the neighborhood at layer lc */
 	HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc);
 
@@ -735,33 +733,14 @@ HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unv
 	*unvisitedLength = 0;
 
 	for (int i = 0; i < localNeighborhood->length; i++)
-		hashes[i] = HnswPtrAccess(base, localNeighborhood->items[i].element)->hash;
-
-	if (base != NULL)
 	{
-		for (int i = 0; i < localNeighborhood->length; i++)
+		HnswCandidate *hc = &localNeighborhood->items[i];
-		{
+		bool		found;
-			HnswCandidate *hc = &localNeighborhood->items[i];
-			bool		found;
 
-			offsethash_insert_hash(v->offsets, HnswPtrOffset(hc->element), hashes[i], &found);
+		AddToVisited(base, v, hc->element, true, &found);
 
-			if (!found)
+		if (!found)
-				unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
+			unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
-		}
-	}
-	else
-	{
-		for (int i = 0; i < localNeighborhood->length; i++)
-		{
-			HnswCandidate *hc = &localNeighborhood->items[i];
-			bool		found;
-
-			pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), hashes[i], &found);
-
-			if (!found)
-				unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
-		}
 	}
 }
 

src/ivfflat.c

@@ -39,16 +39,16 @@ IvfflatInit(void)
 
 	DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
 							"Valid range is 1..lists.", &ivfflat_probes,
-							IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
+							IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	DefineCustomEnumVariable("ivfflat.iterative_scan", "Sets the mode for iterative scans",
 							 NULL, &ivfflat_iterative_scan,
-							 IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
+							 IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	/* If this is less than probes, probes is used */
 	DefineCustomIntVariable("ivfflat.max_probes", "Sets the max number of probes for iterative scans",
 							NULL, &ivfflat_max_probes,
-							IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
+							IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, GUC_EXPLAIN, NULL, NULL, NULL);
 
 	MarkGUCPrefixReserved("ivfflat");
 }

src/ivfscan.c

@@ -114,6 +114,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
 {
 	IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
 	TupleDesc	tupdesc = RelationGetDescr(scan->indexRelation);
+	double		tuples = 0;
 	TupleTableSlot *slot = so->vslot;
 	int			batchProbes = 0;
 
@@ -160,6 +161,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
 				ExecStoreVirtualTuple(slot);
 
 				tuplesort_puttupleslot(so->sortstate, slot);
+
+				tuples++;
 			}
 
 			searchPage = IvfflatPageGetOpaque(page)->nextblkno;
@@ -168,6 +171,12 @@ GetScanItems(IndexScanDesc scan, Datum value)
 		}
 	}
 
+	if (tuples < 100 && ivfflat_iterative_scan == IVFFLAT_ITERATIVE_SCAN_OFF)
+		ereport(DEBUG1,
+				(errmsg("index scan found few tuples"),
+				 errdetail("Index may have been created with little data."),
+				 errhint("Recreate the index and possibly decrease lists.")));
+
 	tuplesort_performsort(so->sortstate);
 
 #if defined(IVFFLAT_MEMORY)

test/expected/hnsw_vector.out

@@ -190,6 +190,4 @@ SHOW hnsw.scan_mem_multiplier;
 
 SET hnsw.scan_mem_multiplier = 0;
 ERROR:  0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
-SET hnsw.scan_mem_multiplier = 1001;
-ERROR:  1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
 DROP TABLE t;

test/sql/hnsw_vector.sql

@@ -109,6 +109,5 @@ SET hnsw.max_scan_tuples = 0;
 SHOW hnsw.scan_mem_multiplier;
 
 SET hnsw.scan_mem_multiplier = 0;
-SET hnsw.scan_mem_multiplier = 1001;
 
 DROP TABLE t;

test/t/043_hnsw_iterative_scan.pl

@@ -56,4 +56,13 @@ foreach ((30000, 50000, 70000))
 	cmp_ok($avg, '<', $expected + 2);
 }
 
+my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
+	SET enable_seqscan = off;
+	SET hnsw.iterative_scan = relaxed_order;
+	SET client_min_messages = debug1;
+	SET work_mem = '1MB';
+	SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
+));
+like($stderr, qr/hnsw index scan reached memory limit after \d+ tuples/);
+
 done_testing();