Allow override

src/hnsw.c

@@ -83,8 +83,8 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	List	   *qinfos;
 #endif
 
-	/* Never use index without order */
+	/* Never use index without order or limit */
-	if (path->indexorderbys == NULL)
+	if (path->indexorderbys == NULL || root->limit_tuples < 0)
 	{
 		*indexStartupCost = DBL_MAX;
 		*indexTotalCost = DBL_MAX;
@@ -95,10 +95,10 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	}
 
 	/*
-	 * Do not use index if no limit or limit + offset > ef_search unless
+	 * Do not use index if limit + offset > ef_search unless enable_seqscan =
-	 * enable_seqscan = off
+	 * off
 	 */
-	if (root->limit_tuples < 0 || root->limit_tuples > hnsw_ef_search)
+	if (root->limit_tuples > hnsw_ef_search)
 	{
 		*indexStartupCost = 1.0e10 - 1;
 		*indexTotalCost = 1.0e10 - 1;

src/ivfflat.c

@@ -76,8 +76,8 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	List	   *qinfos;
 #endif
 
-	/* Never use index without order */
+	/* Never use index without order or limit */
-	if (path->indexorderbys == NULL)
+	if (path->indexorderbys == NULL || root->limit_tuples < 0)
 	{
 		*indexStartupCost = DBL_MAX;
 		*indexTotalCost = DBL_MAX;
@@ -106,10 +106,10 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
 	costs.numIndexTuples = path->indexinfo->tuples * ratio;
 
 	/*
-	 * Do not use index if no limit or limit + offset > expected tuples unless
+	 * Do not use index if limit + offset > expected tuples unless
 	 * enable_seqscan = off
 	 */
-	if (root->limit_tuples < 0 || root->limit_tuples > costs.numIndexTuples)
+	if (root->limit_tuples > costs.numIndexTuples)
 	{
 		*indexStartupCost = 1.0e10 - 1;
 		*indexTotalCost = 1.0e10 - 1;

test/expected/hnsw_cosine.out

@@ -3,7 +3,7 @@ CREATE TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
 INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <=> '[3,3,3]';
+SELECT * FROM t ORDER BY val <=> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,1,1]
@@ -11,13 +11,13 @@ SELECT * FROM t ORDER BY val <=> '[3,3,3]';
  [1,2,4]
 (3 rows)
 
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]' LIMIT 5) t2;
  count 
 -------
      3
 (1 row)
 
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector) LIMIT 5) t2;
  count 
 -------
      3

test/expected/hnsw_ip.out

@@ -3,7 +3,7 @@ CREATE TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING hnsw (val vector_ip_ops);
 INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <#> '[3,3,3]';
+SELECT * FROM t ORDER BY val <#> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,2,4]
@@ -12,7 +12,7 @@ SELECT * FROM t ORDER BY val <#> '[3,3,3]';
  [0,0,0]
 (4 rows)
 
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector) LIMIT 5) t2;
  count 
 -------
      4

test/expected/hnsw_l2.out

@@ -3,7 +3,7 @@ CREATE TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops);
 INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,2,3]
@@ -12,7 +12,7 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
  [0,0,0]
 (4 rows)
 
-SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
+SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector) LIMIT 5;
    val   
 ---------
  [0,0,0]
@@ -28,7 +28,7 @@ SELECT COUNT(*) FROM t;
 (1 row)
 
 TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
  val 
 -----
 (0 rows)

test/expected/hnsw_unlogged.out

@@ -2,7 +2,7 @@ SET enable_seqscan = off;
 CREATE UNLOGGED TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops);
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,2,3]

test/expected/ivfflat_cosine.out

@@ -3,7 +3,7 @@ CREATE TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
 INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <=> '[3,3,3]';
+SELECT * FROM t ORDER BY val <=> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,1,1]
@@ -11,13 +11,13 @@ SELECT * FROM t ORDER BY val <=> '[3,3,3]';
  [1,2,4]
 (3 rows)
 
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]' LIMIT 5) t2;
  count 
 -------
      3
 (1 row)
 
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector) LIMIT 5) t2;
  count 
 -------
      3

test/expected/ivfflat_ip.out

@@ -3,7 +3,7 @@ CREATE TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
 INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <#> '[3,3,3]';
+SELECT * FROM t ORDER BY val <#> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,2,4]
@@ -12,7 +12,7 @@ SELECT * FROM t ORDER BY val <#> '[3,3,3]';
  [0,0,0]
 (4 rows)
 
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector) LIMIT 5) t2;
  count 
 -------
      4

test/expected/ivfflat_l2.out

@@ -3,7 +3,7 @@ CREATE TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
 INSERT INTO t (val) VALUES ('[1,2,4]');
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,2,3]
@@ -12,7 +12,7 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
  [0,0,0]
 (4 rows)
 
-SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
+SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector) LIMIT 5;
    val   
 ---------
  [0,0,0]
@@ -31,7 +31,7 @@ TRUNCATE t;
 NOTICE:  ivfflat index created with little data
 DETAIL:  This will cause low recall.
 HINT:  Drop the index until the table has more data.
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
  val 
 -----
 (0 rows)

test/expected/ivfflat_unlogged.out

@@ -2,7 +2,7 @@ SET enable_seqscan = off;
 CREATE UNLOGGED TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
    val   
 ---------
  [1,2,3]

test/sql/hnsw_cosine.sql

@@ -6,8 +6,8 @@ CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
 
 INSERT INTO t (val) VALUES ('[1,2,4]');
 
-SELECT * FROM t ORDER BY val <=> '[3,3,3]';
+SELECT * FROM t ORDER BY val <=> '[3,3,3]' LIMIT 5;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]' LIMIT 5) t2;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector) LIMIT 5) t2;
 
 DROP TABLE t;

test/sql/hnsw_ip.sql

@@ -6,7 +6,7 @@ CREATE INDEX ON t USING hnsw (val vector_ip_ops);
 
 INSERT INTO t (val) VALUES ('[1,2,4]');
 
-SELECT * FROM t ORDER BY val <#> '[3,3,3]';
+SELECT * FROM t ORDER BY val <#> '[3,3,3]' LIMIT 5;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector) LIMIT 5) t2;
 
 DROP TABLE t;

test/sql/hnsw_l2.sql

@@ -6,11 +6,11 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops);
 
 INSERT INTO t (val) VALUES ('[1,2,4]');
 
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
-SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
+SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector) LIMIT 5;
 SELECT COUNT(*) FROM t;
 
 TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
 
 DROP TABLE t;

test/sql/hnsw_unlogged.sql

@@ -4,6 +4,6 @@ CREATE UNLOGGED TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING hnsw (val vector_l2_ops);
 
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
 
 DROP TABLE t;

test/sql/ivfflat_cosine.sql

@@ -6,8 +6,8 @@ CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
 
 INSERT INTO t (val) VALUES ('[1,2,4]');
 
-SELECT * FROM t ORDER BY val <=> '[3,3,3]';
+SELECT * FROM t ORDER BY val <=> '[3,3,3]' LIMIT 5;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]' LIMIT 5) t2;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector) LIMIT 5) t2;
 
 DROP TABLE t;

test/sql/ivfflat_ip.sql

@@ -6,7 +6,7 @@ CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
 
 INSERT INTO t (val) VALUES ('[1,2,4]');
 
-SELECT * FROM t ORDER BY val <#> '[3,3,3]';
+SELECT * FROM t ORDER BY val <#> '[3,3,3]' LIMIT 5;
-SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
+SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector) LIMIT 5) t2;
 
 DROP TABLE t;

test/sql/ivfflat_l2.sql

@@ -6,11 +6,11 @@ CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
 
 INSERT INTO t (val) VALUES ('[1,2,4]');
 
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
-SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
+SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector) LIMIT 5;
 SELECT COUNT(*) FROM t;
 
 TRUNCATE t;
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
 
 DROP TABLE t;

test/sql/ivfflat_unlogged.sql

@@ -4,6 +4,6 @@ CREATE UNLOGGED TABLE t (val vector(3));
 INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
 CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
 
-SELECT * FROM t ORDER BY val <-> '[3,3,3]';
+SELECT * FROM t ORDER BY val <-> '[3,3,3]' LIMIT 5;
 
 DROP TABLE t;

test/t/007_ivfflat_inserts.pl

@@ -49,7 +49,7 @@ is(idx_scan(), 0);
 $count = $node->safe_psql("postgres", qq(
 	SET enable_seqscan = off;
 	SET ivfflat.probes = 100;
-	SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
+	SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 20000) t;
 ));
 is($count, $expected);
 is(idx_scan(), 1);

test/t/014_hnsw_inserts.pl

@@ -42,7 +42,7 @@ for my $i (1 .. 20)
 
 	my $count = $node->safe_psql("postgres", qq(
 		SET enable_seqscan = off;
-		SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
+		SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 20) t;
 	));
 	is($count, 10);
 
@@ -63,7 +63,7 @@ $node->pgbench(
 my $count = $node->safe_psql("postgres", qq(
 	SET enable_seqscan = off;
 	SET hnsw.ef_search = 1000;
-	SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
+	SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 1000) t;
 ));
 # Elements may lose all incoming connections with the HNSW algorithm
 # Vacuuming can fix this if one of the elements neighbors is deleted

test/t/015_hnsw_duplicates.pl

@@ -26,7 +26,7 @@ sub test_duplicates
 	my $res = $node->safe_psql("postgres", qq(
 		SET enable_seqscan = off;
 		SET hnsw.ef_search = 1;
-		SELECT COUNT(*) FROM (SELECT * FROM tst ORDER BY v <-> '[1,1,1]') t;
+		SELECT COUNT(*) FROM (SELECT * FROM tst ORDER BY v <-> '[1,1,1]' LIMIT 20) t;
 	));
 	is($res, 10);
 }

test/t/018_ivfflat_deletes.pl

@@ -0,0 +1,43 @@
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More;
+
+my $dim = 3;
+
+my $array_sql = join(",", ('random()') x $dim);
+
+# Initialize node
+my $node = get_new_node('node');
+$node->init;
+$node->start;
+
+# Create table and index
+$node->safe_psql("postgres", "CREATE EXTENSION vector;");
+$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
+$node->safe_psql("postgres",
+	"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
+);
+$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
+
+# Delete data
+$node->safe_psql("postgres", "DELETE FROM tst WHERE i % 100 != 0;");
+
+my $exp = $node->safe_psql("postgres", qq(
+	SET enable_indexscan = off;
+	SELECT i FROM tst ORDER BY v <-> '[0,0,0]';
+));
+
+# Run twice to make sure correct tuples marked as dead
+for (1 .. 2)
+{
+	my $res = $node->safe_psql("postgres", qq(
+		SET enable_seqscan = off;
+		SET ivfflat.probes = 100;
+		SELECT i FROM tst ORDER BY v <-> '[0,0,0]';
+	));
+	is($res, $exp);
+}
+
+done_testing();