mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 20:15:46 +08:00
Compare commits
1 Commits
total-cost
...
ivfflat-se
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
61b2ddeb86 |
@@ -1,7 +1,6 @@
|
||||
## 0.8.0 (unreleased)
|
||||
|
||||
- Added casts for arrays to `sparsevec`
|
||||
- Improved cost estimation
|
||||
- Reduced memory usage for HNSW index scans
|
||||
- Dropped support for Postgres 12
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG PG_MAJOR=17
|
||||
ARG PG_MAJOR=16
|
||||
FROM postgres:$PG_MAJOR
|
||||
ARG PG_MAJOR
|
||||
|
||||
|
||||
2
Makefile
2
Makefile
@@ -66,7 +66,7 @@ dist:
|
||||
git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master
|
||||
|
||||
# for Docker
|
||||
PG_MAJOR ?= 17
|
||||
PG_MAJOR ?= 16
|
||||
|
||||
.PHONY: docker
|
||||
|
||||
|
||||
38
README.md
38
README.md
@@ -52,8 +52,6 @@ nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
```
|
||||
|
||||
Note: Postgres 17 is not supported yet due to an upstream issue
|
||||
|
||||
See the [installation notes](#installation-notes---windows) if you run into issues
|
||||
|
||||
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
|
||||
@@ -102,8 +100,6 @@ Or add a vector column to an existing table
|
||||
ALTER TABLE items ADD COLUMN embedding vector(3);
|
||||
```
|
||||
|
||||
Also supports [half-precision](#half-precision-vectors), [binary](#binary-vectors), and [sparse](#sparse-vectors) vectors
|
||||
|
||||
Insert vectors
|
||||
|
||||
```sql
|
||||
@@ -149,8 +145,6 @@ Supported distance functions are:
|
||||
- `<#>` - (negative) inner product
|
||||
- `<=>` - cosine distance
|
||||
- `<+>` - L1 distance (added in 0.7.0)
|
||||
- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
|
||||
- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
|
||||
|
||||
Get the nearest neighbors to a row
|
||||
|
||||
@@ -989,7 +983,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
|
||||
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
|
||||
|
||||
```sh
|
||||
export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
|
||||
export PG_CONFIG=/Library/PostgreSQL/16/bin/pg_config
|
||||
```
|
||||
|
||||
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
|
||||
@@ -1000,11 +994,11 @@ sudo --preserve-env=PG_CONFIG make install
|
||||
|
||||
A few common paths on Mac are:
|
||||
|
||||
- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
|
||||
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
|
||||
- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
|
||||
- EDB installer - `/Library/PostgreSQL/16/bin/pg_config`
|
||||
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@16/bin/pg_config`
|
||||
- Homebrew (x86-64) - `/usr/local/opt/postgresql@16/bin/pg_config`
|
||||
|
||||
Note: Replace `17` with your Postgres server version
|
||||
Note: Replace `16` with your Postgres server version
|
||||
|
||||
### Missing Header
|
||||
|
||||
@@ -1013,10 +1007,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
|
||||
For Ubuntu and Debian, use:
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql-server-dev-17
|
||||
sudo apt install postgresql-server-dev-16
|
||||
```
|
||||
|
||||
Note: Replace `17` with your Postgres server version
|
||||
Note: Replace `16` with your Postgres server version
|
||||
|
||||
### Missing SDK
|
||||
|
||||
@@ -1049,17 +1043,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
|
||||
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
|
||||
|
||||
```sh
|
||||
docker pull pgvector/pgvector:pg17
|
||||
docker pull pgvector/pgvector:pg16
|
||||
```
|
||||
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
|
||||
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
|
||||
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
|
||||
docker build --pull --build-arg PG_MAJOR=16 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
### Homebrew
|
||||
@@ -1070,7 +1064,7 @@ With Homebrew Postgres, you can use:
|
||||
brew install pgvector
|
||||
```
|
||||
|
||||
Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
|
||||
Note: This only adds it to the `postgresql@14` formula
|
||||
|
||||
### PGXN
|
||||
|
||||
@@ -1085,22 +1079,22 @@ pgxn install vector
|
||||
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql-17-pgvector
|
||||
sudo apt install postgresql-16-pgvector
|
||||
```
|
||||
|
||||
Note: Replace `17` with your Postgres server version
|
||||
Note: Replace `16` with your Postgres server version
|
||||
|
||||
### Yum
|
||||
|
||||
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
|
||||
|
||||
```sh
|
||||
sudo yum install pgvector_17
|
||||
sudo yum install pgvector_16
|
||||
# or
|
||||
sudo dnf install pgvector_17
|
||||
sudo dnf install pgvector_16
|
||||
```
|
||||
|
||||
Note: Replace `17` with your Postgres server version
|
||||
Note: Replace `16` with your Postgres server version
|
||||
|
||||
### pkg
|
||||
|
||||
|
||||
@@ -159,6 +159,24 @@ CheckStateArray(ArrayType *statearray, const char *caller)
|
||||
return (float8 *) ARR_DATA_PTR(statearray);
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM < 120003
|
||||
static pg_noinline void
|
||||
float_overflow_error(void)
|
||||
{
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("value out of range: overflow")));
|
||||
}
|
||||
|
||||
static pg_noinline void
|
||||
float_underflow_error(void)
|
||||
{
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("value out of range: underflow")));
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Convert textual representation to internal representation
|
||||
*/
|
||||
|
||||
71
src/hnsw.c
71
src/hnsw.c
@@ -12,7 +12,6 @@
|
||||
#include "utils/float.h"
|
||||
#include "utils/guc.h"
|
||||
#include "utils/selfuncs.h"
|
||||
#include "utils/spccache.h"
|
||||
|
||||
#if PG_VERSION_NUM < 150000
|
||||
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
|
||||
@@ -100,9 +99,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
{
|
||||
GenericCosts costs;
|
||||
int m;
|
||||
double ratio;
|
||||
double startupPages;
|
||||
double spc_seq_page_cost;
|
||||
int entryLevel;
|
||||
Relation index;
|
||||
|
||||
/* Never use index without order */
|
||||
@@ -118,71 +115,21 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
|
||||
MemSet(&costs, 0, sizeof(costs));
|
||||
|
||||
genericcostestimate(root, path, loop_count, &costs);
|
||||
|
||||
index = index_open(path->indexinfo->indexoid, NoLock);
|
||||
HnswGetMetaPageInfo(index, &m, NULL);
|
||||
index_close(index, NoLock);
|
||||
|
||||
/*
|
||||
* HNSW cost estimation follows a formula that accounts for the total
|
||||
* number of tuples indexed combined with the parameters that most
|
||||
* influence the duration of the index scan, namely: m - the number of
|
||||
* tuples that are scanned in each step of the HNSW graph traversal
|
||||
* ef_search - which influences the total number of steps taken at layer 0
|
||||
*
|
||||
* The source of the vector data can impact how many steps it takes to
|
||||
* converge on the set of vectors to return to the executor. Currently, we
|
||||
* use a hardcoded scaling factor (HNSWScanScalingFactor) to help
|
||||
* influence that, but this could later become a configurable parameter
|
||||
* based on the cost estimations.
|
||||
*
|
||||
* The tuple estimator formula is below:
|
||||
*
|
||||
* numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
|
||||
*
|
||||
* "entryLevel * m" represents the floor of tuples we need to scan to get
|
||||
* to layer 0 (L0).
|
||||
*
|
||||
* "layer0TuplesMax" is the estimated total number of tuples we'd scan at
|
||||
* L0 if we weren't discarding already visited tuples as part of the scan.
|
||||
*
|
||||
* "layer0Selectivity" estimates the percentage of tuples that are scanned
|
||||
* at L0, accounting for previously visited tuples, multiplied by the
|
||||
* "scalingFactor" (currently hardcoded).
|
||||
*/
|
||||
if (path->indexinfo->tuples > 0)
|
||||
{
|
||||
double scalingFactor = 0.55;
|
||||
int entryLevel = (int) (log(path->indexinfo->tuples) * HnswGetMl(m));
|
||||
int layer0TuplesMax = HnswGetLayerM(m, 0) * hnsw_ef_search;
|
||||
double layer0Selectivity = scalingFactor * log(path->indexinfo->tuples) / (log(m) * (1 + log(hnsw_ef_search)));
|
||||
/* Approximate entry level */
|
||||
entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
|
||||
|
||||
ratio = (entryLevel * m + layer0TuplesMax * layer0Selectivity) / path->indexinfo->tuples;
|
||||
/* TODO Improve estimate of visited tuples (currently underestimates) */
|
||||
/* Account for number of tuples (or entry level), m, and ef_search */
|
||||
costs.numIndexTuples = (entryLevel + 2) * m;
|
||||
|
||||
if (ratio > 1)
|
||||
ratio = 1;
|
||||
}
|
||||
else
|
||||
ratio = 1;
|
||||
genericcostestimate(root, path, loop_count, &costs);
|
||||
|
||||
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
|
||||
|
||||
/* Startup cost is cost before returning the first row */
|
||||
costs.indexStartupCost = costs.indexTotalCost * ratio;
|
||||
|
||||
/* Adjust cost if needed since TOAST not included in seq scan cost */
|
||||
startupPages = costs.numIndexPages * ratio;
|
||||
if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
|
||||
{
|
||||
/* Change all page cost from random to sequential */
|
||||
costs.indexStartupCost -= startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
|
||||
/* Remove cost of extra pages */
|
||||
costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
|
||||
}
|
||||
|
||||
*indexStartupCost = costs.indexStartupCost;
|
||||
/* Use total cost since most work happens before first tuple is returned */
|
||||
*indexStartupCost = costs.indexTotalCost;
|
||||
*indexTotalCost = costs.indexTotalCost;
|
||||
*indexSelectivity = costs.indexSelectivity;
|
||||
*indexCorrelation = costs.indexCorrelation;
|
||||
|
||||
@@ -759,16 +759,9 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
|
||||
page = BufferGetPage(buf);
|
||||
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
|
||||
|
||||
/* Ensure expected neighbors */
|
||||
if (ntup->count != (element->level + 2) * m)
|
||||
{
|
||||
UnlockReleaseBuffer(buf);
|
||||
return;
|
||||
}
|
||||
start = (element->level - lc) * m;
|
||||
|
||||
/* Copy to minimize lock time */
|
||||
start = (element->level - lc) * m;
|
||||
memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
@@ -57,6 +57,39 @@ ivfflatbuildphasename(int64 phasenum)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Estimate the number of probes needed to get requested tuples
|
||||
*/
|
||||
static int
|
||||
EstimateProbes(PlannerInfo *root, IndexPath *path, int lists)
|
||||
{
|
||||
double selectivity = 1;
|
||||
ListCell *lc;
|
||||
double tuplesPerList;
|
||||
|
||||
/* Cannot estimate without limit */
|
||||
/* limit_tuples includes offset */
|
||||
if (root->limit_tuples < 0)
|
||||
return 0;
|
||||
|
||||
/* Get the selectivity of non-index conditions */
|
||||
foreach(lc, path->indexinfo->indrestrictinfo)
|
||||
{
|
||||
RestrictInfo *rinfo = lfirst(lc);
|
||||
|
||||
/* Skip DEFAULT_INEQ_SEL since it may be a distance filter */
|
||||
if (rinfo->norm_selec >= 0 && rinfo->norm_selec <= 1 && rinfo->norm_selec != (Selectivity) DEFAULT_INEQ_SEL)
|
||||
selectivity *= rinfo->norm_selec;
|
||||
}
|
||||
|
||||
tuplesPerList = path->indexinfo->tuples * selectivity / (double) lists;
|
||||
if (tuplesPerList == 0)
|
||||
return lists;
|
||||
|
||||
/* No need to cap at this point */
|
||||
return root->limit_tuples / tuplesPerList;
|
||||
}
|
||||
|
||||
/*
|
||||
* Estimate the cost of an index scan
|
||||
*/
|
||||
@@ -68,9 +101,8 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
{
|
||||
GenericCosts costs;
|
||||
int lists;
|
||||
int probes;
|
||||
double ratio;
|
||||
double sequentialRatio = 0.5;
|
||||
double startupPages;
|
||||
double spc_seq_page_cost;
|
||||
Relation index;
|
||||
|
||||
@@ -87,37 +119,52 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
|
||||
MemSet(&costs, 0, sizeof(costs));
|
||||
|
||||
genericcostestimate(root, path, loop_count, &costs);
|
||||
|
||||
index = index_open(path->indexinfo->indexoid, NoLock);
|
||||
IvfflatGetMetaPageInfo(index, &lists, NULL);
|
||||
index_close(index, NoLock);
|
||||
|
||||
probes = Max(ivfflat_probes, EstimateProbes(root, path, lists));
|
||||
|
||||
/* Get the ratio of lists that we need to visit */
|
||||
ratio = ((double) ivfflat_probes) / lists;
|
||||
ratio = ((double) probes) / lists;
|
||||
if (ratio > 1.0)
|
||||
ratio = 1.0;
|
||||
|
||||
/*
|
||||
* This gives us the subset of tuples to visit. This value is passed into
|
||||
* the generic cost estimator to determine the number of pages to visit
|
||||
* during the index scan.
|
||||
*/
|
||||
costs.numIndexTuples = path->indexinfo->tuples * ratio;
|
||||
|
||||
genericcostestimate(root, path, loop_count, &costs);
|
||||
|
||||
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
|
||||
|
||||
/* Change some page cost from random to sequential */
|
||||
costs.indexTotalCost -= sequentialRatio * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
|
||||
/* Startup cost is cost before returning the first row */
|
||||
costs.indexStartupCost = costs.indexTotalCost * ratio;
|
||||
|
||||
/* Adjust cost if needed since TOAST not included in seq scan cost */
|
||||
startupPages = costs.numIndexPages * ratio;
|
||||
if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
|
||||
if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
|
||||
{
|
||||
/* Change rest of page cost from random to sequential */
|
||||
costs.indexStartupCost -= (1 - sequentialRatio) * startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
/* Change all page cost from random to sequential */
|
||||
costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
|
||||
/* Remove cost of extra pages */
|
||||
costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
|
||||
costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Change some page cost from random to sequential */
|
||||
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
}
|
||||
|
||||
*indexStartupCost = costs.indexStartupCost;
|
||||
/*
|
||||
* If the list selectivity is lower than what is returned from the generic
|
||||
* cost estimator, use that.
|
||||
*/
|
||||
if (ratio < costs.indexSelectivity)
|
||||
costs.indexSelectivity = ratio;
|
||||
|
||||
/* Use total cost since most work happens before first tuple is returned */
|
||||
*indexStartupCost = costs.indexTotalCost;
|
||||
*indexTotalCost = costs.indexTotalCost;
|
||||
*indexSelectivity = costs.indexSelectivity;
|
||||
*indexCorrelation = costs.indexCorrelation;
|
||||
|
||||
18
src/vector.c
18
src/vector.c
@@ -155,6 +155,24 @@ CheckStateArray(ArrayType *statearray, const char *caller)
|
||||
return (float8 *) ARR_DATA_PTR(statearray);
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM < 120003
|
||||
static pg_noinline void
|
||||
float_overflow_error(void)
|
||||
{
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("value out of range: overflow")));
|
||||
}
|
||||
|
||||
static pg_noinline void
|
||||
float_underflow_error(void)
|
||||
{
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("value out of range: underflow")));
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Convert textual representation to internal representation
|
||||
*/
|
||||
|
||||
@@ -5,7 +5,7 @@ use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $dim = 3;
|
||||
my $nc = 50;
|
||||
my $nc = 100;
|
||||
my $limit = 20;
|
||||
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
@@ -94,15 +94,15 @@ like($explain, qr/Seq Scan/);
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query';
|
||||
));
|
||||
like($explain, qr/Seq Scan/);
|
||||
# TODO Do not use index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test attribute index
|
||||
$node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
# TODO Use attribute index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
like($explain, qr/Index Scan on attribute_idx/);
|
||||
|
||||
# Test partial index
|
||||
$node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5) WHERE (c = $c);");
|
||||
|
||||
@@ -18,13 +18,9 @@ $node->start;
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);");
|
||||
$node->safe_psql("postgres", "CREATE TABLE cat (i int4 PRIMARY KEY, t text, b boolean);");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO cat SELECT i, 'cat ' || i, i % 5 = 0 FROM generate_series(1, $nc) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
@@ -41,7 +37,8 @@ my $c = int(rand() * $nc);
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Seq Scan/);
|
||||
# TODO Do not use index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test attribute filtering with few rows removed
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
@@ -59,7 +56,8 @@ like($explain, qr/Index Scan using idx/);
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE c < 1 ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Seq Scan/);
|
||||
# TODO Do not use index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test attribute filtering with few rows removed like
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
@@ -98,25 +96,13 @@ $explain = $node->safe_psql("postgres", qq(
|
||||
));
|
||||
like($explain, qr/Seq Scan/);
|
||||
|
||||
# Test join
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test join with attribute filtering
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c WHERE cat.b = 't' ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test attribute index
|
||||
$node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
# Use attribute index
|
||||
like($explain, qr/Bitmap Index Scan on attribute_idx/);
|
||||
# TODO Use attribute index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test partial index
|
||||
$node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);");
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my @dims = (384, 1536);
|
||||
my $limit = 10;
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
|
||||
for my $dim (@dims)
|
||||
{
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 2000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
# Generate query
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
my $query = "[" . join(",", @r) . "]";
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# 3x the rows are needed for distance filters
|
||||
# since the planner uses DEFAULT_INEQ_SEL for the selectivity (should be 1)
|
||||
# Recreate index for performance
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(2001, 6000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP TABLE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,50 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my @dims = (384, 1536);
|
||||
my $limit = 10;
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
|
||||
for my $dim (@dims)
|
||||
{
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 5000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
# Generate query
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
my $query = "[" . join(",", @r) . "]";
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP TABLE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
Reference in New Issue
Block a user