mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
2 Commits
dynamic-pr
...
parallel-i
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54870b9bdc | ||
|
|
0950968c11 |
2
.github/workflows/build.yml
vendored
2
.github/workflows/build.yml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
- postgres: 12
|
||||
os: ubuntu-20.04
|
||||
- postgres: 11
|
||||
os: ubuntu-20.04
|
||||
os: ubuntu-18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ankane/setup-postgres@v1
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
## 0.4.2 (unreleased)
|
||||
|
||||
- Added notice when index created with little data
|
||||
- Fixed installation error with Postgres 12.0-12.2
|
||||
|
||||
## 0.4.1 (2023-03-21)
|
||||
|
||||
- Improved performance of cosine distance
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
ARG PG_MAJOR=15
|
||||
FROM postgres:$PG_MAJOR
|
||||
ARG PG_MAJOR
|
||||
FROM postgres:15
|
||||
|
||||
COPY . /tmp/pgvector
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends build-essential postgresql-server-dev-$PG_MAJOR && \
|
||||
apt-get install -y --no-install-recommends build-essential postgresql-server-dev-15 && \
|
||||
cd /tmp/pgvector && \
|
||||
make clean && \
|
||||
make OPTFLAGS="" && \
|
||||
@@ -13,6 +11,6 @@ RUN apt-get update && \
|
||||
mkdir /usr/share/doc/pgvector && \
|
||||
cp LICENSE README.md /usr/share/doc/pgvector && \
|
||||
rm -r /tmp/pgvector && \
|
||||
apt-get remove -y build-essential postgresql-server-dev-$PG_MAJOR && \
|
||||
apt-get remove -y build-essential postgresql-server-dev-15 && \
|
||||
apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
1
Makefile
1
Makefile
@@ -14,7 +14,6 @@ OPTFLAGS = -march=native
|
||||
# Mac ARM doesn't support -march=native
|
||||
ifeq ($(shell uname -s), Darwin)
|
||||
ifeq ($(shell uname -p), arm)
|
||||
# no difference with -march=armv8.5-a
|
||||
OPTFLAGS =
|
||||
endif
|
||||
endif
|
||||
|
||||
224
README.md
224
README.md
@@ -2,11 +2,13 @@
|
||||
|
||||
Open-source vector similarity search for Postgres
|
||||
|
||||
Supports
|
||||
```sql
|
||||
CREATE TABLE items (embedding vector(3));
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops);
|
||||
SELECT * FROM items ORDER BY embedding <-> '[1,2,3]' LIMIT 5;
|
||||
```
|
||||
|
||||
- exact and approximate nearest neighbor search
|
||||
- L2 distance, inner product, and cosine distance
|
||||
- any [language](#languages) with a Postgres client
|
||||
Supports L2 distance, inner product, and cosine distance
|
||||
|
||||
[](https://github.com/pgvector/pgvector/actions)
|
||||
|
||||
@@ -15,7 +17,6 @@ Supports
|
||||
Compile and install the extension (supports Postgres 11+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
@@ -28,88 +29,41 @@ Then load it in databases where you want to use it
|
||||
CREATE EXTENSION vector;
|
||||
```
|
||||
|
||||
See the [installation notes](#installation-notes) if you run into issues
|
||||
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [Yum](#yum), or [conda-forge](#conda-forge)
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), or [conda-forge](#conda-forge)
|
||||
|
||||
## Getting Started
|
||||
|
||||
Create a vector column with 3 dimensions
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
|
||||
CREATE TABLE items (embedding vector(3));
|
||||
```
|
||||
|
||||
Insert vectors
|
||||
Insert values
|
||||
|
||||
```sql
|
||||
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
|
||||
INSERT INTO items VALUES ('[1,2,3]'), ('[4,5,6]');
|
||||
```
|
||||
|
||||
Get the nearest neighbors by L2 distance
|
||||
Get the nearest neighbor by L2 distance
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 1;
|
||||
```
|
||||
|
||||
Also supports inner product (`<#>`) and cosine distance (`<=>`)
|
||||
|
||||
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
|
||||
|
||||
## Storing
|
||||
|
||||
Create a new table with a vector column
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
|
||||
```
|
||||
|
||||
Or add a vector column to an existing table
|
||||
|
||||
```sql
|
||||
ALTER TABLE items ADD COLUMN embedding vector(3);
|
||||
```
|
||||
|
||||
Insert vectors
|
||||
|
||||
```sql
|
||||
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
|
||||
```
|
||||
|
||||
Upsert vectors
|
||||
|
||||
```sql
|
||||
INSERT INTO items (id, embedding) VALUES (1, '[1,2,3]'), (2, '[4,5,6]')
|
||||
ON CONFLICT (id) DO UPDATE SET embedding = EXCLUDED.embedding;
|
||||
```
|
||||
|
||||
Update vectors
|
||||
|
||||
```sql
|
||||
UPDATE items SET embedding = '[1,2,3]' WHERE id = 1;
|
||||
```
|
||||
|
||||
Delete vectors
|
||||
|
||||
```sql
|
||||
DELETE FROM items WHERE id = 1;
|
||||
```
|
||||
|
||||
## Querying
|
||||
|
||||
Get the nearest neighbors to a vector
|
||||
Use a `SELECT` clause to get the distance
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
SELECT embedding <-> '[3,1,2]' AS distance FROM items;
|
||||
```
|
||||
|
||||
Get the nearest neighbors to a row
|
||||
|
||||
```sql
|
||||
SELECT * FROM items WHERE id != 1 ORDER BY embedding <-> (SELECT embedding FROM items WHERE id = 1) LIMIT 5;
|
||||
```
|
||||
|
||||
Get rows within a certain distance
|
||||
Use a `WHERE` clause to get rows within a certain distance
|
||||
|
||||
```sql
|
||||
SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5;
|
||||
@@ -117,80 +71,55 @@ SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5;
|
||||
|
||||
Note: Combine with `ORDER BY` and `LIMIT` to use an index
|
||||
|
||||
#### Distances
|
||||
|
||||
Get the distance
|
||||
|
||||
```sql
|
||||
SELECT embedding <-> '[3,1,2]' AS distance FROM items;
|
||||
```
|
||||
|
||||
For inner product, multiply by -1 (since `<#>` returns the negative inner product)
|
||||
|
||||
```sql
|
||||
SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items;
|
||||
```
|
||||
|
||||
For cosine similarity, use 1 - cosine distance
|
||||
|
||||
```sql
|
||||
SELECT 1 - (embedding <=> '[3,1,2]') AS cosine_similarity FROM items;
|
||||
```
|
||||
|
||||
#### Aggregates
|
||||
|
||||
Average vectors
|
||||
Get the average of vectors
|
||||
|
||||
```sql
|
||||
SELECT AVG(embedding) FROM items;
|
||||
```
|
||||
|
||||
Average groups of vectors
|
||||
|
||||
```sql
|
||||
SELECT category_id, AVG(embedding) FROM items GROUP BY category_id;
|
||||
```
|
||||
|
||||
## Indexing
|
||||
|
||||
By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
|
||||
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for performance. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Three keys to achieving good recall are:
|
||||
|
||||
1. Create the index *after* the table has some data
|
||||
2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows
|
||||
3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `lists / 10` for up to 1M rows and `sqrt(lists)` for over 1M rows
|
||||
|
||||
Add an index for each distance function you want to use.
|
||||
Speed up queries with an approximate index. Add an index for each distance function you want to use.
|
||||
|
||||
L2 distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops);
|
||||
```
|
||||
|
||||
Inner product
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops) WITH (lists = 100);
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops);
|
||||
```
|
||||
|
||||
Cosine distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
Indexes should be created after the table has some data for optimal clustering. Also, unlike typical indexes which only affect performance, you may see different results for queries after adding an approximate index. Vectors with up to 2,000 dimensions can be indexed.
|
||||
|
||||
### Index Options
|
||||
|
||||
Specify the number of inverted lists (100 by default)
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
A lower value provides better recall at the cost of speed. A good place to start is:
|
||||
|
||||
- `rows / 1000` for up to 1M rows
|
||||
- `sqrt(rows)` for over 1M rows
|
||||
|
||||
### Query Options
|
||||
|
||||
Specify the number of probes (1 by default)
|
||||
|
||||
```sql
|
||||
SET ivfflat.probes = 10;
|
||||
SET ivfflat.probes = 1;
|
||||
```
|
||||
|
||||
A higher value provides better recall at the cost of speed, and it can be set to the number of lists for exact nearest neighbor search (at which point the planner won’t use the index)
|
||||
@@ -199,7 +128,7 @@ Use `SET LOCAL` inside a transaction to set it for a single query
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
SET LOCAL ivfflat.probes = 10;
|
||||
SET LOCAL ivfflat.probes = 1;
|
||||
SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
@@ -232,7 +161,7 @@ SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIM
|
||||
can be indexed with:
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) WHERE (category_id = 123);
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WHERE (category_id = 123);
|
||||
```
|
||||
|
||||
To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`.
|
||||
@@ -243,34 +172,24 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
|
||||
|
||||
## Performance
|
||||
|
||||
Use `EXPLAIN ANALYZE` to debug performance.
|
||||
|
||||
```sql
|
||||
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
### Exact Search
|
||||
|
||||
To speed up queries without an index, increase `max_parallel_workers_per_gather`.
|
||||
|
||||
```sql
|
||||
SET max_parallel_workers_per_gather = 4;
|
||||
```
|
||||
|
||||
If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance.
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
### Approximate Search
|
||||
|
||||
To speed up queries with an index, increase the number of inverted lists (at the expense of recall).
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
```
|
||||
|
||||
Use `EXPLAIN ANALYZE` to debug performance.
|
||||
|
||||
```sql
|
||||
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 1;
|
||||
```
|
||||
|
||||
## Languages
|
||||
|
||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||
@@ -279,10 +198,8 @@ Language | Libraries / Examples
|
||||
--- | ---
|
||||
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
|
||||
C# | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
|
||||
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
|
||||
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
|
||||
Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
|
||||
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
|
||||
Java, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
|
||||
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
|
||||
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
|
||||
@@ -293,7 +210,6 @@ Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
|
||||
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
|
||||
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
|
||||
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
|
||||
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
@@ -312,10 +228,6 @@ Two things you can try are:
|
||||
1. use dimensionality reduction
|
||||
2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
|
||||
|
||||
#### Why am I seeing less results after adding an index?
|
||||
|
||||
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
||||
|
||||
## Reference
|
||||
|
||||
### Vector Type
|
||||
@@ -348,42 +260,6 @@ Function | Description
|
||||
--- | ---
|
||||
avg(vector) → vector | arithmetic mean
|
||||
|
||||
## Installation Notes
|
||||
|
||||
### Postgres Location
|
||||
|
||||
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
|
||||
|
||||
```sh
|
||||
export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
|
||||
```
|
||||
|
||||
Then re-run the installation instructions (run `make clean` before `make` if needed)
|
||||
|
||||
### Missing Header
|
||||
|
||||
If compilation fails with `fatal error: postgres.h: No such file or directory`, make sure Postgres development files are installed on the server.
|
||||
|
||||
For Ubuntu and Debian, use:
|
||||
|
||||
```sh
|
||||
sudo apt-get install postgresql-server-dev-15
|
||||
```
|
||||
|
||||
Note: Replace `15` with your Postgres server version
|
||||
|
||||
### Windows
|
||||
|
||||
Support for Windows is currently experimental. Use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\15"
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
```
|
||||
|
||||
## Additional Installation Methods
|
||||
|
||||
### Docker
|
||||
@@ -420,18 +296,6 @@ Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) wi
|
||||
pgxn install vector
|
||||
```
|
||||
|
||||
### Yum
|
||||
|
||||
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
|
||||
|
||||
```sh
|
||||
sudo yum install pgvector_15
|
||||
# or
|
||||
sudo dnf install pgvector_15
|
||||
```
|
||||
|
||||
Note: Replace `15` with your Postgres server version
|
||||
|
||||
### conda-forge
|
||||
|
||||
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
|
||||
@@ -452,7 +316,7 @@ To request a new extension on other providers:
|
||||
- Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
|
||||
- Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
|
||||
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
|
||||
- Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
|
||||
- Render - vote or comment on [this page](https://feedback.render.com/features/p/add-pgvector-extension-to-postgresql)
|
||||
|
||||
## Upgrading
|
||||
|
||||
|
||||
@@ -431,18 +431,8 @@ ComputeCenters(IvfflatBuildState * buildstate)
|
||||
/* TODO Ensure within maintenance_work_mem */
|
||||
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions);
|
||||
if (buildstate->heap != NULL)
|
||||
{
|
||||
SampleRows(buildstate);
|
||||
|
||||
if (buildstate->samples->length < buildstate->lists)
|
||||
{
|
||||
ereport(NOTICE,
|
||||
(errmsg("ivfflat index created with little data"),
|
||||
errdetail("this will cause poor recall"),
|
||||
errhint("drop the index until the table has more data")));
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate centers */
|
||||
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
|
||||
|
||||
|
||||
@@ -103,7 +103,13 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
if (ratio > 1)
|
||||
ratio = 1;
|
||||
|
||||
// cost estimates for parallel workers applied outside of amcostestimate
|
||||
elog(INFO, "parallel_workers = %d, parallel aware = %d", path->path.parallel_workers, path->path.parallel_aware);
|
||||
|
||||
costs.indexTotalCost *= ratio;
|
||||
costs.numIndexPages *= ratio;
|
||||
|
||||
elog(INFO, "ivfflatcostestimate = %f", costs.indexTotalCost);
|
||||
|
||||
/* Startup cost and total cost are same */
|
||||
*indexStartupCost = costs.indexTotalCost;
|
||||
@@ -151,6 +157,25 @@ ivfflatvalidate(Oid opclassoid)
|
||||
return true;
|
||||
}
|
||||
|
||||
static Size
|
||||
ivfflatestimateparallelscan()
|
||||
{
|
||||
elog(INFO, "ivfflatestimateparallelscan");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
ivfflatinitparallelscan(void *target)
|
||||
{
|
||||
elog(INFO, "ivfflatinitparallelscan");
|
||||
}
|
||||
|
||||
static void
|
||||
ivfflatparallelrescan(IndexScanDesc scan)
|
||||
{
|
||||
elog(INFO, "ivfflatparallelrescan");
|
||||
}
|
||||
|
||||
/*
|
||||
* Define index handler
|
||||
*
|
||||
@@ -178,7 +203,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amstorage = false;
|
||||
amroutine->amclusterable = false;
|
||||
amroutine->ampredlocks = false;
|
||||
amroutine->amcanparallel = false;
|
||||
amroutine->amcanparallel = true;
|
||||
amroutine->amcaninclude = false;
|
||||
#if PG_VERSION_NUM >= 130000
|
||||
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
|
||||
@@ -212,9 +237,9 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amrestrpos = NULL;
|
||||
|
||||
/* Interface functions to support parallel index scans */
|
||||
amroutine->amestimateparallelscan = NULL;
|
||||
amroutine->aminitparallelscan = NULL;
|
||||
amroutine->amparallelrescan = NULL;
|
||||
amroutine->amestimateparallelscan = ivfflatestimateparallelscan;
|
||||
amroutine->aminitparallelscan = ivfflatinitparallelscan;
|
||||
amroutine->amparallelrescan = ivfflatparallelrescan;
|
||||
|
||||
PG_RETURN_POINTER(amroutine);
|
||||
}
|
||||
|
||||
@@ -206,7 +206,6 @@ typedef struct IvfflatScanOpaqueData
|
||||
|
||||
/* Lists */
|
||||
pairingheap *listQueue;
|
||||
double minDistance;
|
||||
IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */
|
||||
} IvfflatScanOpaqueData;
|
||||
|
||||
|
||||
@@ -60,9 +60,6 @@ GetScanLists(IndexScanDesc scan, Datum value)
|
||||
/* Use procinfo from the index instead of scan key for performance */
|
||||
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
|
||||
|
||||
if (distance < so->minDistance)
|
||||
so->minDistance = distance;
|
||||
|
||||
if (listCount < so->probes)
|
||||
{
|
||||
scanlist = &so->lists[listCount];
|
||||
@@ -114,7 +111,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
Datum datum;
|
||||
bool isnull;
|
||||
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
|
||||
double tuples = 0;
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
|
||||
@@ -132,13 +128,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
/* Search closest probes lists */
|
||||
while (!pairingheap_is_empty(so->listQueue))
|
||||
{
|
||||
IvfflatScanList *scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
|
||||
|
||||
/* Query-aware dynamic pruning */
|
||||
if (scanlist->distance > 2 * so->minDistance)
|
||||
continue;
|
||||
|
||||
searchPage = scanlist->startPage;
|
||||
searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
|
||||
|
||||
/* Search all entry pages for list */
|
||||
while (BlockNumberIsValid(searchPage))
|
||||
@@ -169,8 +159,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
ExecStoreVirtualTuple(slot);
|
||||
|
||||
tuplesort_puttupleslot(so->sortstate, slot);
|
||||
|
||||
tuples++;
|
||||
}
|
||||
|
||||
searchPage = IvfflatPageGetOpaque(page)->nextblkno;
|
||||
@@ -179,13 +167,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO Scan more lists */
|
||||
if (tuples < 100)
|
||||
ereport(DEBUG1,
|
||||
(errmsg("index scan found few tuples"),
|
||||
errdetail("index may have been created without data or lists is too high"),
|
||||
errhint("recreate the index and possibly decrease lists")));
|
||||
|
||||
tuplesort_performsort(so->sortstate);
|
||||
}
|
||||
|
||||
@@ -261,7 +242,6 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
|
||||
|
||||
so->first = true;
|
||||
pairingheap_reset(so->listQueue);
|
||||
so->minDistance = DBL_MAX;
|
||||
|
||||
if (keys && scan->numberOfKeys > 0)
|
||||
memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
|
||||
|
||||
13
src/vector.c
13
src/vector.c
@@ -100,7 +100,7 @@ CheckStateArray(ArrayType *statearray, const char *caller)
|
||||
return (float8 *) ARR_DATA_PTR(statearray);
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM < 120003
|
||||
#if PG_VERSION_NUM < 120000
|
||||
static pg_noinline void
|
||||
float_overflow_error(void)
|
||||
{
|
||||
@@ -467,7 +467,6 @@ l2_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
diff = ax[i] - bx[i];
|
||||
@@ -494,7 +493,6 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
diff = ax[i] - bx[i];
|
||||
@@ -519,7 +517,6 @@ inner_product(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += ax[i] * bx[i];
|
||||
|
||||
@@ -541,7 +538,6 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += ax[i] * bx[i];
|
||||
|
||||
@@ -565,7 +561,6 @@ cosine_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
distance += ax[i] * bx[i];
|
||||
@@ -592,7 +587,6 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += a->x[i] * b->x[i];
|
||||
|
||||
@@ -628,7 +622,6 @@ vector_norm(PG_FUNCTION_ARGS)
|
||||
float *ax = a->x;
|
||||
double norm = 0.0;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
norm += ax[i] * ax[i];
|
||||
|
||||
@@ -653,8 +646,6 @@ vector_add(PG_FUNCTION_ARGS)
|
||||
|
||||
result = InitVector(a->dim);
|
||||
rx = result->x;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] + bx[i];
|
||||
|
||||
@@ -679,8 +670,6 @@ vector_sub(PG_FUNCTION_ARGS)
|
||||
|
||||
result = InitVector(a->dim);
|
||||
rx = result->x;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] - bx[i];
|
||||
|
||||
|
||||
15
test/sql/ivfflat_parallel.sql
Normal file
15
test/sql/ivfflat_parallel.sql
Normal file
@@ -0,0 +1,15 @@
|
||||
-- SET force_parallel_mode = on;
|
||||
SET parallel_setup_cost = 10;
|
||||
SET parallel_tuple_cost = 0.001;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
SET min_parallel_index_scan_size = 1;
|
||||
|
||||
CREATE TABLE t (id integer, val vector(3));
|
||||
INSERT INTO t (id, val) SELECT n, ARRAY[random(), random(), random()] FROM generate_series(1,1000000) n;
|
||||
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 10);
|
||||
SET ivfflat.probes = 2;
|
||||
|
||||
EXPLAIN SELECT * FROM t ORDER BY val <-> '[0.5,0.5,0.5]' LIMIT 5;
|
||||
SELECT * FROM t ORDER BY val <-> '[0.5,0.5,0.5]' LIMIT 5;
|
||||
|
||||
DROP TABLE t;
|
||||
Reference in New Issue
Block a user