Compare commits

..

64 Commits

Author SHA1 Message Date
Andrew Kane
7132c9b111 Use fabs [skip ci] 2023-05-05 18:34:15 -07:00
Andrew Kane
878e1e6c3a Added query-aware dynamic pruning [skip ci] 2023-05-05 18:13:56 -07:00
Andrew Kane
d885e2bcfa Added FAQ about results [skip ci] 2023-05-02 10:17:31 -07:00
Andrew Kane
a445355a48 Added Heroku Postgres link [skip ci] 2023-04-26 12:48:55 -07:00
Andrew Kane
d5b17a3624 Fixed installation error with Postgres 12.0-12.2 - fixes #101 2023-04-25 09:36:21 -07:00
Andrew Kane
6383078029 Updated readme [skip ci] 2023-04-22 17:07:10 -07:00
Andrew Kane
5146c7cc57 Added guidance for probes to readme [skip ci] 2023-04-20 12:32:49 -07:00
Andrew Kane
ac63f9858b Updated readme [skip ci] 2023-04-14 12:35:59 -07:00
Andrew Kane
76a4166857 Added link to pgvector-crystal [skip ci] 2023-04-13 21:10:18 -07:00
Andrew Kane
31fb6963a3 Now available on Render [skip ci] 2023-04-13 13:20:06 -07:00
Andrew Kane
18c06cb9b1 Added link to pgvector-swift [skip ci] 2023-04-11 21:11:46 -07:00
Andrew Kane
f858796c64 Added link to pgvector-haskell [skip ci] 2023-04-11 12:53:00 -07:00
Andrew Kane
f32f695844 Improved notice [skip ci] 2023-04-10 21:31:33 -07:00
Andrew Kane
1b013a94f7 Added notice when index created with little data [skip ci] 2023-04-10 21:28:24 -07:00
Andrew Kane
00148dfa1f Improved indexing docs [skip ci] 2023-04-10 21:12:25 -07:00
Andrew Kane
67fc791d95 Improved indexing docs [skip ci] 2023-04-10 21:04:46 -07:00
Andrew Kane
8bf360ed84 Updated header [skip ci] 2023-04-02 11:24:46 -07:00
Andrew Kane
f79d28347b Updated installation notes [skip ci] 2023-04-02 11:21:57 -07:00
Andrew Kane
20cf63de0a Updated readme [skip ci] 2023-04-02 10:56:37 -07:00
Andrew Kane
587cbcf15b Updated readme [skip ci] 2023-04-02 10:46:44 -07:00
Andrew Kane
c09edb5b8f Moved instructions [skip ci] 2023-04-02 10:39:41 -07:00
Will Laurance
c63501cca4 Update readme to show example usage of PG_CONFIG (#79) 2023-04-02 10:34:02 -07:00
Andrew Kane
58f0c922d2 Updated readme [skip ci] 2023-04-02 10:24:21 -07:00
Andrew Kane
36e73d2818 Updated readme [skip ci] 2023-04-01 20:06:18 -07:00
Andrew Kane
dd92d0ece3 Updated readme [skip ci] 2023-04-01 20:04:47 -07:00
Andrew Kane
6ede7681a5 Updated readme [skip ci] 2023-04-01 19:47:13 -07:00
Andrew Kane
8733729149 Updated readme [skip ci] 2023-04-01 19:45:13 -07:00
Andrew Kane
03a5789132 Updated readme [skip ci] 2023-04-01 19:44:23 -07:00
Andrew Kane
e5b612a856 Updated readme [skip ci] 2023-04-01 19:34:21 -07:00
Andrew Kane
9a7d3532f5 Updated readme [skip ci] 2023-04-01 19:32:26 -07:00
Andrew Kane
91315dfeff Added additional instructions for Ubuntu, Debian, and Windows [skip ci] 2023-04-01 17:42:44 -07:00
Andrew Kane
6e3101d527 Updated Dockerfile [skip ci] 2023-04-01 13:31:52 -07:00
Andrew Kane
96ae1a6a72 Added PG_MAJOR arg to Dockerfile [skip ci] 2023-04-01 13:15:00 -07:00
Andrew Kane
55aeba8bd6 Added lists to example [skip ci] 2023-03-31 22:12:35 -07:00
Andrew Kane
aebe1bae02 Updated readme [skip ci] 2023-03-31 21:55:24 -07:00
Andrew Kane
14355b9312 Updated readme [skip ci] 2023-03-31 21:46:38 -07:00
Andrew Kane
b5c66d0416 Updated readme [skip ci] 2023-03-31 21:36:25 -07:00
Andrew Kane
f534d9878a Updated readme [skip ci] 2023-03-31 21:31:38 -07:00
Andrew Kane
f3df137db6 Improved indexing instructions [skip ci] 2023-03-31 21:28:45 -07:00
Andrew Kane
489cdb5068 Added delete example [skip ci] 2023-03-31 20:02:22 -07:00
Andrew Kane
161f48793e Updated readme [skip ci] 2023-03-31 19:59:53 -07:00
Andrew Kane
f0f7ffca41 Updated readme [skip ci] 2023-03-31 19:34:39 -07:00
Andrew Kane
138d9be616 Added storage examples [skip ci] 2023-03-31 19:17:26 -07:00
Andrew Kane
fb98e73255 Updated readme [skip ci] 2023-03-31 18:55:51 -07:00
Andrew Kane
4754cac40c Updated readme [skip ci] 2023-03-31 18:47:15 -07:00
Andrew Kane
8432efb7d8 Added cosine similarity example [skip ci] 2023-03-31 18:34:36 -07:00
Andrew Kane
c38410259c Updated header [skip ci] 2023-03-31 16:59:21 -07:00
Andrew Kane
609d9fbf0a Updated header [skip ci] 2023-03-31 16:54:14 -07:00
Andrew Kane
7946424639 Updated header [skip ci] 2023-03-31 16:49:40 -07:00
Andrew Kane
d51310dfa0 Updated readme [skip ci] 2023-03-31 16:38:09 -07:00
Andrew Kane
30f2893aeb Updated readme [skip ci] 2023-03-31 16:30:42 -07:00
Andrew Kane
5d0f88529e Updated readme [skip ci] 2023-03-31 16:28:27 -07:00
Andrew Kane
1d020abdd1 Added auto-vectorized comments [skip ci] 2023-03-31 16:20:10 -07:00
Andrew Kane
d0fdd42652 Added comment [skip ci] 2023-03-31 13:37:33 -07:00
Andrew Kane
8473468925 Added instructions for Yum - #76 2023-03-30 15:39:27 -07:00
Andrew Kane
9c01524466 Updated readme [skip ci] 2023-03-26 23:39:09 -07:00
Andrew Kane
121baa411e Added debug message for index scan 2023-03-26 12:45:44 -07:00
Andrew Kane
50005d7326 Updated CI [skip ci] 2023-03-26 09:46:21 -07:00
Andrew Kane
ec12d79cbc Added link to pgvector-perl [skip ci] 2023-03-23 21:42:43 -07:00
Andrew Kane
d3eb56df07 Improved tests 2023-03-22 16:35:56 -07:00
Andrew Kane
13f7aa50c3 Added Render link [skip ci] 2023-03-22 16:29:06 -07:00
Andrew Kane
81e9e72fbc Updated guidance on lists [skip ci] 2023-03-22 14:00:47 -07:00
Andrew Kane
8d95510302 Updated readme [skip ci] 2023-03-22 13:32:25 -07:00
Andrew Kane
53bb2ed0cd Updated Homebrew instructions [skip ci] 2023-03-21 13:20:57 -07:00
11 changed files with 283 additions and 70 deletions

View File

@@ -17,7 +17,7 @@ jobs:
- postgres: 12 - postgres: 12
os: ubuntu-20.04 os: ubuntu-20.04
- postgres: 11 - postgres: 11
os: ubuntu-18.04 os: ubuntu-20.04
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1

View File

@@ -1,3 +1,8 @@
## 0.4.2 (unreleased)
- Added notice when index created with little data
- Fixed installation error with Postgres 12.0-12.2
## 0.4.1 (2023-03-21) ## 0.4.1 (2023-03-21)
- Improved performance of cosine distance - Improved performance of cosine distance

View File

@@ -1,9 +1,11 @@
FROM postgres:15 ARG PG_MAJOR=15
FROM postgres:$PG_MAJOR
ARG PG_MAJOR
COPY . /tmp/pgvector COPY . /tmp/pgvector
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y --no-install-recommends build-essential postgresql-server-dev-15 && \ apt-get install -y --no-install-recommends build-essential postgresql-server-dev-$PG_MAJOR && \
cd /tmp/pgvector && \ cd /tmp/pgvector && \
make clean && \ make clean && \
make OPTFLAGS="" && \ make OPTFLAGS="" && \
@@ -11,6 +13,6 @@ RUN apt-get update && \
mkdir /usr/share/doc/pgvector && \ mkdir /usr/share/doc/pgvector && \
cp LICENSE README.md /usr/share/doc/pgvector && \ cp LICENSE README.md /usr/share/doc/pgvector && \
rm -r /tmp/pgvector && \ rm -r /tmp/pgvector && \
apt-get remove -y build-essential postgresql-server-dev-15 && \ apt-get remove -y build-essential postgresql-server-dev-$PG_MAJOR && \
apt-get autoremove -y && \ apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*

View File

@@ -14,6 +14,7 @@ OPTFLAGS = -march=native
# Mac ARM doesn't support -march=native # Mac ARM doesn't support -march=native
ifeq ($(shell uname -s), Darwin) ifeq ($(shell uname -s), Darwin)
ifeq ($(shell uname -p), arm) ifeq ($(shell uname -p), arm)
# no difference with -march=armv8.5-a
OPTFLAGS = OPTFLAGS =
endif endif
endif endif

247
README.md
View File

@@ -2,13 +2,11 @@
Open-source vector similarity search for Postgres Open-source vector similarity search for Postgres
```sql Supports
CREATE TABLE items (embedding vector(3));
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops);
SELECT * FROM items ORDER BY embedding <-> '[1,2,3]' LIMIT 5;
```
Supports L2 distance, inner product, and cosine distance - exact and approximate nearest neighbor search
- L2 distance, inner product, and cosine distance
- any [language](#languages) with a Postgres client
[![Build Status](https://github.com/pgvector/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector/actions) [![Build Status](https://github.com/pgvector/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector/actions)
@@ -17,6 +15,7 @@ Supports L2 distance, inner product, and cosine distance
Compile and install the extension (supports Postgres 11+) Compile and install the extension (supports Postgres 11+)
```sh ```sh
cd /tmp
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
@@ -29,41 +28,88 @@ Then load it in databases where you want to use it
CREATE EXTENSION vector; CREATE EXTENSION vector;
``` ```
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), or [conda-forge](#conda-forge) See the [installation notes](#installation-notes) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [Yum](#yum), or [conda-forge](#conda-forge)
## Getting Started ## Getting Started
Create a vector column with 3 dimensions Create a vector column with 3 dimensions
```sql ```sql
CREATE TABLE items (embedding vector(3)); CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
``` ```
Insert values Insert vectors
```sql ```sql
INSERT INTO items VALUES ('[1,2,3]'), ('[4,5,6]'); INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
``` ```
Get the nearest neighbor by L2 distance Get the nearest neighbors by L2 distance
```sql ```sql
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 1; SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
``` ```
Also supports inner product (`<#>`) and cosine distance (`<=>`) Also supports inner product (`<#>`) and cosine distance (`<=>`)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
## Querying ## Storing
Use a `SELECT` clause to get the distance Create a new table with a vector column
```sql ```sql
SELECT embedding <-> '[3,1,2]' AS distance FROM items; CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
``` ```
Use a `WHERE` clause to get rows within a certain distance Or add a vector column to an existing table
```sql
ALTER TABLE items ADD COLUMN embedding vector(3);
```
Insert vectors
```sql
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
```
Upsert vectors
```sql
INSERT INTO items (id, embedding) VALUES (1, '[1,2,3]'), (2, '[4,5,6]')
ON CONFLICT (id) DO UPDATE SET embedding = EXCLUDED.embedding;
```
Update vectors
```sql
UPDATE items SET embedding = '[1,2,3]' WHERE id = 1;
```
Delete vectors
```sql
DELETE FROM items WHERE id = 1;
```
## Querying
Get the nearest neighbors to a vector
```sql
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
Get the nearest neighbors to a row
```sql
SELECT * FROM items WHERE id != 1 ORDER BY embedding <-> (SELECT embedding FROM items WHERE id = 1) LIMIT 5;
```
Get rows within a certain distance
```sql ```sql
SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5; SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5;
@@ -71,61 +117,89 @@ SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5;
Note: Combine with `ORDER BY` and `LIMIT` to use an index Note: Combine with `ORDER BY` and `LIMIT` to use an index
Get the average of vectors #### Distances
Get the distance
```sql
SELECT embedding <-> '[3,1,2]' AS distance FROM items;
```
For inner product, multiply by -1 (since `<#>` returns the negative inner product)
```sql
SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items;
```
For cosine similarity, use 1 - cosine distance
```sql
SELECT 1 - (embedding <=> '[3,1,2]') AS cosine_similarity FROM items;
```
#### Aggregates
Average vectors
```sql ```sql
SELECT AVG(embedding) FROM items; SELECT AVG(embedding) FROM items;
``` ```
Average groups of vectors
```sql
SELECT category_id, AVG(embedding) FROM items GROUP BY category_id;
```
## Indexing ## Indexing
Speed up queries with an approximate index. Add an index for each distance function you want to use. By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
You can add an index to use approximate nearest neighbor search, which trades some recall for performance. Unlike typical indexes, you will see different results for queries after adding an approximate index.
Three keys to achieving good recall are:
1. Create the index *after* the table has some data
2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows
3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `lists / 10` for up to 1M rows and `sqrt(lists)` for over 1M rows
Add an index for each distance function you want to use.
L2 distance L2 distance
```sql
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops);
```
Inner product
```sql
CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops);
```
Cosine distance
```sql
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops);
```
Indexes should be created after the table has some data for optimal clustering. Also, unlike typical indexes which only affect performance, you may see different results for queries after adding an approximate index. Vectors with up to 2,000 dimensions can be indexed.
### Index Options
Specify the number of inverted lists (100 by default)
```sql ```sql
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
``` ```
A [good place to start](https://github.com/facebookresearch/faiss/issues/112) is `4 * sqrt(rows)` Inner product
```sql
CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops) WITH (lists = 100);
```
Cosine distance
```sql
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
```
Vectors with up to 2,000 dimensions can be indexed.
### Query Options ### Query Options
Specify the number of probes (1 by default) Specify the number of probes (1 by default)
```sql ```sql
SET ivfflat.probes = 1; SET ivfflat.probes = 10;
``` ```
A higher value improves recall at the cost of speed, and it can be set to the number of lists for exact nearest neighbor search (at which point the planner wont use the index) A higher value provides better recall at the cost of speed, and it can be set to the number of lists for exact nearest neighbor search (at which point the planner wont use the index)
Use `SET LOCAL` inside a transaction to set it for a single query Use `SET LOCAL` inside a transaction to set it for a single query
```sql ```sql
BEGIN; BEGIN;
SET LOCAL ivfflat.probes = 1; SET LOCAL ivfflat.probes = 10;
SELECT ... SELECT ...
COMMIT; COMMIT;
``` ```
@@ -158,7 +232,7 @@ SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIM
can be indexed with: can be indexed with:
```sql ```sql
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WHERE (category_id = 123); CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) WHERE (category_id = 123);
``` ```
To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`. To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`.
@@ -169,24 +243,34 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
## Performance ## Performance
Use `EXPLAIN ANALYZE` to debug performance.
```sql
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
### Exact Search
To speed up queries without an index, increase `max_parallel_workers_per_gather`. To speed up queries without an index, increase `max_parallel_workers_per_gather`.
```sql ```sql
SET max_parallel_workers_per_gather = 4; SET max_parallel_workers_per_gather = 4;
``` ```
If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance.
```sql
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
```
### Approximate Search
To speed up queries with an index, increase the number of inverted lists (at the expense of recall). To speed up queries with an index, increase the number of inverted lists (at the expense of recall).
```sql ```sql
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000); CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
``` ```
Use `EXPLAIN ANALYZE` to debug performance.
```sql
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 1;
```
## Languages ## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another. Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
@@ -195,17 +279,21 @@ Language | Libraries / Examples
--- | --- --- | ---
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp) C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C# | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet) C# | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir) Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
Go | [pgvector-go](https://github.com/pgvector/pgvector-go) Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java) Java, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia) Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua) Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
Node.js | [pgvector-node](https://github.com/pgvector/pgvector-node) Node.js | [pgvector-node](https://github.com/pgvector/pgvector-node)
Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php) PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python) Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r) R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor) Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust) Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
## Frequently Asked Questions ## Frequently Asked Questions
@@ -224,6 +312,10 @@ Two things you can try are:
1. use dimensionality reduction 1. use dimensionality reduction
2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h` 2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
#### Why am I seeing less results after adding an index?
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
## Reference ## Reference
### Vector Type ### Vector Type
@@ -256,6 +348,42 @@ Function | Description
--- | --- --- | ---
avg(vector) → vector | arithmetic mean avg(vector) → vector | arithmetic mean
## Installation Notes
### Postgres Location
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
```sh
export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
```
Then re-run the installation instructions (run `make clean` before `make` if needed)
### Missing Header
If compilation fails with `fatal error: postgres.h: No such file or directory`, make sure Postgres development files are installed on the server.
For Ubuntu and Debian, use:
```sh
sudo apt-get install postgresql-server-dev-15
```
Note: Replace `15` with your Postgres server version
### Windows
Support for Windows is currently experimental. Use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\15"
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
```
## Additional Installation Methods ## Additional Installation Methods
### Docker ### Docker
@@ -281,7 +409,7 @@ docker build -t pgvector .
With Homebrew Postgres, you can use: With Homebrew Postgres, you can use:
```sh ```sh
brew install pgvector/brew/pgvector brew install pgvector
``` ```
### PGXN ### PGXN
@@ -292,6 +420,18 @@ Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) wi
pgxn install vector pgxn install vector
``` ```
### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
```sh
sudo yum install pgvector_15
# or
sudo dnf install pgvector_15
```
Note: Replace `15` with your Postgres server version
### conda-forge ### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with: With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
@@ -310,8 +450,9 @@ To request a new extension on other providers:
- Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/) - Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
- Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065) - Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
- Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307) - Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
- Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
## Upgrading ## Upgrading

View File

@@ -431,8 +431,18 @@ ComputeCenters(IvfflatBuildState * buildstate)
/* TODO Ensure within maintenance_work_mem */ /* TODO Ensure within maintenance_work_mem */
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions); buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions);
if (buildstate->heap != NULL) if (buildstate->heap != NULL)
{
SampleRows(buildstate); SampleRows(buildstate);
if (buildstate->samples->length < buildstate->lists)
{
ereport(NOTICE,
(errmsg("ivfflat index created with little data"),
errdetail("this will cause poor recall"),
errhint("drop the index until the table has more data")));
}
}
/* Calculate centers */ /* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers)); IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));

View File

@@ -206,6 +206,7 @@ typedef struct IvfflatScanOpaqueData
/* Lists */ /* Lists */
pairingheap *listQueue; pairingheap *listQueue;
double minDistance;
IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */ IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */
} IvfflatScanOpaqueData; } IvfflatScanOpaqueData;

View File

@@ -60,6 +60,9 @@ GetScanLists(IndexScanDesc scan, Datum value)
/* Use procinfo from the index instead of scan key for performance */ /* Use procinfo from the index instead of scan key for performance */
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value)); distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
if (distance < so->minDistance)
so->minDistance = distance;
if (listCount < so->probes) if (listCount < so->probes)
{ {
scanlist = &so->lists[listCount]; scanlist = &so->lists[listCount];
@@ -111,6 +114,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
Datum datum; Datum datum;
bool isnull; bool isnull;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation); TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
double tuples = 0;
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual); TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
@@ -128,7 +132,13 @@ GetScanItems(IndexScanDesc scan, Datum value)
/* Search closest probes lists */ /* Search closest probes lists */
while (!pairingheap_is_empty(so->listQueue)) while (!pairingheap_is_empty(so->listQueue))
{ {
searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage; IvfflatScanList *scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
/* Query-aware dynamic pruning */
if (fabs(scanlist->distance) > 1.5 * fabs(so->minDistance))
continue;
searchPage = scanlist->startPage;
/* Search all entry pages for list */ /* Search all entry pages for list */
while (BlockNumberIsValid(searchPage)) while (BlockNumberIsValid(searchPage))
@@ -159,6 +169,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
ExecStoreVirtualTuple(slot); ExecStoreVirtualTuple(slot);
tuplesort_puttupleslot(so->sortstate, slot); tuplesort_puttupleslot(so->sortstate, slot);
tuples++;
} }
searchPage = IvfflatPageGetOpaque(page)->nextblkno; searchPage = IvfflatPageGetOpaque(page)->nextblkno;
@@ -167,6 +179,13 @@ GetScanItems(IndexScanDesc scan, Datum value)
} }
} }
/* TODO Scan more lists */
if (tuples < 100)
ereport(DEBUG1,
(errmsg("index scan found few tuples"),
errdetail("index may have been created without data or lists is too high"),
errhint("recreate the index and possibly decrease lists")));
tuplesort_performsort(so->sortstate); tuplesort_performsort(so->sortstate);
} }
@@ -242,6 +261,7 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
so->first = true; so->first = true;
pairingheap_reset(so->listQueue); pairingheap_reset(so->listQueue);
so->minDistance = DBL_MAX;
if (keys && scan->numberOfKeys > 0) if (keys && scan->numberOfKeys > 0)
memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData)); memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));

View File

@@ -100,7 +100,7 @@ CheckStateArray(ArrayType *statearray, const char *caller)
return (float8 *) ARR_DATA_PTR(statearray); return (float8 *) ARR_DATA_PTR(statearray);
} }
#if PG_VERSION_NUM < 120000 #if PG_VERSION_NUM < 120003
static pg_noinline void static pg_noinline void
float_overflow_error(void) float_overflow_error(void)
{ {
@@ -467,6 +467,7 @@ l2_distance(PG_FUNCTION_ARGS)
CheckDims(a, b); CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
diff = ax[i] - bx[i]; diff = ax[i] - bx[i];
@@ -493,6 +494,7 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
CheckDims(a, b); CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
diff = ax[i] - bx[i]; diff = ax[i] - bx[i];
@@ -517,6 +519,7 @@ inner_product(PG_FUNCTION_ARGS)
CheckDims(a, b); CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
distance += ax[i] * bx[i]; distance += ax[i] * bx[i];
@@ -538,6 +541,7 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
CheckDims(a, b); CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
distance += ax[i] * bx[i]; distance += ax[i] * bx[i];
@@ -561,6 +565,7 @@ cosine_distance(PG_FUNCTION_ARGS)
CheckDims(a, b); CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
distance += ax[i] * bx[i]; distance += ax[i] * bx[i];
@@ -587,6 +592,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
CheckDims(a, b); CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
distance += a->x[i] * b->x[i]; distance += a->x[i] * b->x[i];
@@ -622,6 +628,7 @@ vector_norm(PG_FUNCTION_ARGS)
float *ax = a->x; float *ax = a->x;
double norm = 0.0; double norm = 0.0;
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
norm += ax[i] * ax[i]; norm += ax[i] * ax[i];
@@ -646,6 +653,8 @@ vector_add(PG_FUNCTION_ARGS)
result = InitVector(a->dim); result = InitVector(a->dim);
rx = result->x; rx = result->x;
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++) for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] + bx[i]; rx[i] = ax[i] + bx[i];
@@ -670,6 +679,8 @@ vector_sub(PG_FUNCTION_ARGS)
result = InitVector(a->dim); result = InitVector(a->dim);
rx = result->x; rx = result->x;
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++) for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] - bx[i]; rx[i] = ax[i] - bx[i];

View File

@@ -22,10 +22,28 @@ SELECT round(vector_norm('[1,1]')::numeric, 5);
1.41421 1.41421
(1 row) (1 row)
SELECT round(l2_distance('[1,2]', '[0,0]')::numeric, 5); SELECT vector_norm('[3,4]');
round vector_norm
--------- -------------
2.23607 5
(1 row)
SELECT vector_norm('[0,1]');
vector_norm
-------------
1
(1 row)
SELECT l2_distance('[0,0]', '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]', '[0,1]');
l2_distance
-------------
1
(1 row) (1 row)
SELECT l2_distance('[1,2]', '[3]'); SELECT l2_distance('[1,2]', '[3]');
@@ -38,10 +56,10 @@ SELECT inner_product('[1,2]', '[3,4]');
SELECT inner_product('[1,2]', '[3]'); SELECT inner_product('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1 ERROR: different vector dimensions 2 and 1
SELECT round(cosine_distance('[1,2]', '[2,4]')::numeric, 5); SELECT cosine_distance('[1,2]', '[2,4]');
round cosine_distance
--------- -----------------
0.00000 0
(1 row) (1 row)
SELECT cosine_distance('[1,2]', '[0,0]'); SELECT cosine_distance('[1,2]', '[0,0]');

View File

@@ -2,15 +2,19 @@ SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[1,2,3]'::vector - '[4,5,6]'; SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT vector_dims('[1,2,3]'); SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT round(l2_distance('[1,2]', '[0,0]')::numeric, 5); SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT vector_norm('[3,4]');
SELECT vector_norm('[0,1]');
SELECT l2_distance('[0,0]', '[3,4]');
SELECT l2_distance('[0,0]', '[0,1]');
SELECT l2_distance('[1,2]', '[3]'); SELECT l2_distance('[1,2]', '[3]');
SELECT inner_product('[1,2]', '[3,4]'); SELECT inner_product('[1,2]', '[3,4]');
SELECT inner_product('[1,2]', '[3]'); SELECT inner_product('[1,2]', '[3]');
SELECT round(cosine_distance('[1,2]', '[2,4]')::numeric, 5); SELECT cosine_distance('[1,2]', '[2,4]');
SELECT cosine_distance('[1,2]', '[0,0]'); SELECT cosine_distance('[1,2]', '[0,0]');
SELECT cosine_distance('[1,1]', '[1,1]'); SELECT cosine_distance('[1,1]', '[1,1]');
SELECT cosine_distance('[1,1]', '[-1,-1]'); SELECT cosine_distance('[1,1]', '[-1,-1]');