mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
70 Commits
parallel-i
...
v0.4.2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69672cd84d | ||
|
|
300adba2f1 | ||
|
|
e362279199 | ||
|
|
53301021f6 | ||
|
|
8f589f6d09 | ||
|
|
dcf206128a | ||
|
|
3244d40e8a | ||
|
|
7d8dbcaa3c | ||
|
|
7f575f55fb | ||
|
|
94e7487d5f | ||
|
|
74a3cd597f | ||
|
|
db8ed738b8 | ||
|
|
54c550420b | ||
|
|
cc539a0a27 | ||
|
|
d885e2bcfa | ||
|
|
a445355a48 | ||
|
|
d5b17a3624 | ||
|
|
6383078029 | ||
|
|
5146c7cc57 | ||
|
|
ac63f9858b | ||
|
|
76a4166857 | ||
|
|
31fb6963a3 | ||
|
|
18c06cb9b1 | ||
|
|
f858796c64 | ||
|
|
f32f695844 | ||
|
|
1b013a94f7 | ||
|
|
00148dfa1f | ||
|
|
67fc791d95 | ||
|
|
8bf360ed84 | ||
|
|
f79d28347b | ||
|
|
20cf63de0a | ||
|
|
587cbcf15b | ||
|
|
c09edb5b8f | ||
|
|
c63501cca4 | ||
|
|
58f0c922d2 | ||
|
|
36e73d2818 | ||
|
|
dd92d0ece3 | ||
|
|
6ede7681a5 | ||
|
|
8733729149 | ||
|
|
03a5789132 | ||
|
|
e5b612a856 | ||
|
|
9a7d3532f5 | ||
|
|
91315dfeff | ||
|
|
6e3101d527 | ||
|
|
96ae1a6a72 | ||
|
|
55aeba8bd6 | ||
|
|
aebe1bae02 | ||
|
|
14355b9312 | ||
|
|
b5c66d0416 | ||
|
|
f534d9878a | ||
|
|
f3df137db6 | ||
|
|
489cdb5068 | ||
|
|
161f48793e | ||
|
|
f0f7ffca41 | ||
|
|
138d9be616 | ||
|
|
fb98e73255 | ||
|
|
4754cac40c | ||
|
|
8432efb7d8 | ||
|
|
c38410259c | ||
|
|
609d9fbf0a | ||
|
|
7946424639 | ||
|
|
d51310dfa0 | ||
|
|
30f2893aeb | ||
|
|
5d0f88529e | ||
|
|
1d020abdd1 | ||
|
|
d0fdd42652 | ||
|
|
8473468925 | ||
|
|
9c01524466 | ||
|
|
121baa411e | ||
|
|
50005d7326 |
2
.github/workflows/build.yml
vendored
2
.github/workflows/build.yml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
- postgres: 12
|
||||
os: ubuntu-20.04
|
||||
- postgres: 11
|
||||
os: ubuntu-18.04
|
||||
os: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ankane/setup-postgres@v1
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
## 0.4.2 (2023-05-13)
|
||||
|
||||
- Added notice when index created with little data
|
||||
- Fixed dimensions check for some direct function calls
|
||||
- Fixed installation error with Postgres 12.0-12.2
|
||||
|
||||
## 0.4.1 (2023-03-21)
|
||||
|
||||
- Improved performance of cosine distance
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
FROM postgres:15
|
||||
ARG PG_MAJOR=15
|
||||
FROM postgres:$PG_MAJOR
|
||||
ARG PG_MAJOR
|
||||
|
||||
COPY . /tmp/pgvector
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends build-essential postgresql-server-dev-15 && \
|
||||
apt-get install -y --no-install-recommends build-essential postgresql-server-dev-$PG_MAJOR && \
|
||||
cd /tmp/pgvector && \
|
||||
make clean && \
|
||||
make OPTFLAGS="" && \
|
||||
@@ -11,6 +13,6 @@ RUN apt-get update && \
|
||||
mkdir /usr/share/doc/pgvector && \
|
||||
cp LICENSE README.md /usr/share/doc/pgvector && \
|
||||
rm -r /tmp/pgvector && \
|
||||
apt-get remove -y build-essential postgresql-server-dev-15 && \
|
||||
apt-get remove -y build-essential postgresql-server-dev-$PG_MAJOR && \
|
||||
apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.2",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.2",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
9
Makefile
9
Makefile
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.4.1
|
||||
EXTVERSION = 0.4.2
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
@@ -14,6 +14,7 @@ OPTFLAGS = -march=native
|
||||
# Mac ARM doesn't support -march=native
|
||||
ifeq ($(shell uname -s), Darwin)
|
||||
ifeq ($(shell uname -p), arm)
|
||||
# no difference with -march=armv8.5-a
|
||||
OPTFLAGS =
|
||||
endif
|
||||
endif
|
||||
@@ -62,3 +63,9 @@ dist:
|
||||
|
||||
docker:
|
||||
docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest .
|
||||
|
||||
.PHONY: docker-release
|
||||
|
||||
docker-release:
|
||||
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 -t ankane/pgvector:latest .
|
||||
docker buildx build --push --platform linux/amd64,linux/arm64 -t ankane/pgvector:v$(EXTVERSION) .
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.4.1
|
||||
EXTVERSION = 0.4.2
|
||||
|
||||
OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||
|
||||
|
||||
258
README.md
258
README.md
@@ -2,13 +2,11 @@
|
||||
|
||||
Open-source vector similarity search for Postgres
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (embedding vector(3));
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops);
|
||||
SELECT * FROM items ORDER BY embedding <-> '[1,2,3]' LIMIT 5;
|
||||
```
|
||||
Supports
|
||||
|
||||
Supports L2 distance, inner product, and cosine distance
|
||||
- exact and approximate nearest neighbor search
|
||||
- L2 distance, inner product, and cosine distance
|
||||
- any [language](#languages) with a Postgres client
|
||||
|
||||
[](https://github.com/pgvector/pgvector/actions)
|
||||
|
||||
@@ -17,7 +15,8 @@ Supports L2 distance, inner product, and cosine distance
|
||||
Compile and install the extension (supports Postgres 11+)
|
||||
|
||||
```sh
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
cd /tmp
|
||||
git clone --branch v0.4.2 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -29,41 +28,88 @@ Then load it in databases where you want to use it
|
||||
CREATE EXTENSION vector;
|
||||
```
|
||||
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), or [conda-forge](#conda-forge)
|
||||
See the [installation notes](#installation-notes) if you run into issues
|
||||
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [Yum](#yum), or [conda-forge](#conda-forge)
|
||||
|
||||
## Getting Started
|
||||
|
||||
Create a vector column with 3 dimensions
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (embedding vector(3));
|
||||
CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
|
||||
```
|
||||
|
||||
Insert values
|
||||
Insert vectors
|
||||
|
||||
```sql
|
||||
INSERT INTO items VALUES ('[1,2,3]'), ('[4,5,6]');
|
||||
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
|
||||
```
|
||||
|
||||
Get the nearest neighbor by L2 distance
|
||||
Get the nearest neighbors by L2 distance
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 1;
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
Also supports inner product (`<#>`) and cosine distance (`<=>`)
|
||||
|
||||
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
|
||||
|
||||
## Querying
|
||||
## Storing
|
||||
|
||||
Use a `SELECT` clause to get the distance
|
||||
Create a new table with a vector column
|
||||
|
||||
```sql
|
||||
SELECT embedding <-> '[3,1,2]' AS distance FROM items;
|
||||
CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
|
||||
```
|
||||
|
||||
Use a `WHERE` clause to get rows within a certain distance
|
||||
Or add a vector column to an existing table
|
||||
|
||||
```sql
|
||||
ALTER TABLE items ADD COLUMN embedding vector(3);
|
||||
```
|
||||
|
||||
Insert vectors
|
||||
|
||||
```sql
|
||||
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
|
||||
```
|
||||
|
||||
Upsert vectors
|
||||
|
||||
```sql
|
||||
INSERT INTO items (id, embedding) VALUES (1, '[1,2,3]'), (2, '[4,5,6]')
|
||||
ON CONFLICT (id) DO UPDATE SET embedding = EXCLUDED.embedding;
|
||||
```
|
||||
|
||||
Update vectors
|
||||
|
||||
```sql
|
||||
UPDATE items SET embedding = '[1,2,3]' WHERE id = 1;
|
||||
```
|
||||
|
||||
Delete vectors
|
||||
|
||||
```sql
|
||||
DELETE FROM items WHERE id = 1;
|
||||
```
|
||||
|
||||
## Querying
|
||||
|
||||
Get the nearest neighbors to a vector
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
Get the nearest neighbors to a row
|
||||
|
||||
```sql
|
||||
SELECT * FROM items WHERE id != 1 ORDER BY embedding <-> (SELECT embedding FROM items WHERE id = 1) LIMIT 5;
|
||||
```
|
||||
|
||||
Get rows within a certain distance
|
||||
|
||||
```sql
|
||||
SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5;
|
||||
@@ -71,55 +117,80 @@ SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5;
|
||||
|
||||
Note: Combine with `ORDER BY` and `LIMIT` to use an index
|
||||
|
||||
Get the average of vectors
|
||||
#### Distances
|
||||
|
||||
Get the distance
|
||||
|
||||
```sql
|
||||
SELECT embedding <-> '[3,1,2]' AS distance FROM items;
|
||||
```
|
||||
|
||||
For inner product, multiply by -1 (since `<#>` returns the negative inner product)
|
||||
|
||||
```sql
|
||||
SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items;
|
||||
```
|
||||
|
||||
For cosine similarity, use 1 - cosine distance
|
||||
|
||||
```sql
|
||||
SELECT 1 - (embedding <=> '[3,1,2]') AS cosine_similarity FROM items;
|
||||
```
|
||||
|
||||
#### Aggregates
|
||||
|
||||
Average vectors
|
||||
|
||||
```sql
|
||||
SELECT AVG(embedding) FROM items;
|
||||
```
|
||||
|
||||
Average groups of vectors
|
||||
|
||||
```sql
|
||||
SELECT category_id, AVG(embedding) FROM items GROUP BY category_id;
|
||||
```
|
||||
|
||||
## Indexing
|
||||
|
||||
Speed up queries with an approximate index. Add an index for each distance function you want to use.
|
||||
By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
|
||||
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for performance. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Three keys to achieving good recall are:
|
||||
|
||||
1. Create the index *after* the table has some data
|
||||
2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows
|
||||
3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `lists / 10` for up to 1M rows and `sqrt(lists)` for over 1M rows
|
||||
|
||||
Add an index for each distance function you want to use.
|
||||
|
||||
L2 distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops);
|
||||
```
|
||||
|
||||
Inner product
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops);
|
||||
```
|
||||
|
||||
Cosine distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
Indexes should be created after the table has some data for optimal clustering. Also, unlike typical indexes which only affect performance, you may see different results for queries after adding an approximate index. Vectors with up to 2,000 dimensions can be indexed.
|
||||
|
||||
### Index Options
|
||||
|
||||
Specify the number of inverted lists (100 by default)
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
A lower value provides better recall at the cost of speed. A good place to start is:
|
||||
Inner product
|
||||
|
||||
- `rows / 1000` for up to 1M rows
|
||||
- `sqrt(rows)` for over 1M rows
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
Cosine distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
|
||||
### Query Options
|
||||
|
||||
Specify the number of probes (1 by default)
|
||||
|
||||
```sql
|
||||
SET ivfflat.probes = 1;
|
||||
SET ivfflat.probes = 10;
|
||||
```
|
||||
|
||||
A higher value provides better recall at the cost of speed, and it can be set to the number of lists for exact nearest neighbor search (at which point the planner won’t use the index)
|
||||
@@ -128,7 +199,7 @@ Use `SET LOCAL` inside a transaction to set it for a single query
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
SET LOCAL ivfflat.probes = 1;
|
||||
SET LOCAL ivfflat.probes = 10;
|
||||
SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
@@ -161,7 +232,7 @@ SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIM
|
||||
can be indexed with:
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WHERE (category_id = 123);
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) WHERE (category_id = 123);
|
||||
```
|
||||
|
||||
To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`.
|
||||
@@ -172,24 +243,34 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
|
||||
|
||||
## Performance
|
||||
|
||||
Use `EXPLAIN ANALYZE` to debug performance.
|
||||
|
||||
```sql
|
||||
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
### Exact Search
|
||||
|
||||
To speed up queries without an index, increase `max_parallel_workers_per_gather`.
|
||||
|
||||
```sql
|
||||
SET max_parallel_workers_per_gather = 4;
|
||||
```
|
||||
|
||||
If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance.
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
### Approximate Search
|
||||
|
||||
To speed up queries with an index, increase the number of inverted lists (at the expense of recall).
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
```
|
||||
|
||||
Use `EXPLAIN ANALYZE` to debug performance.
|
||||
|
||||
```sql
|
||||
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 1;
|
||||
```
|
||||
|
||||
## Languages
|
||||
|
||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||
@@ -198,8 +279,10 @@ Language | Libraries / Examples
|
||||
--- | ---
|
||||
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
|
||||
C# | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
|
||||
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
|
||||
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
|
||||
Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
|
||||
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
|
||||
Java, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
|
||||
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
|
||||
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
|
||||
@@ -210,6 +293,7 @@ Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
|
||||
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
|
||||
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
|
||||
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
|
||||
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
@@ -223,10 +307,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
|
||||
|
||||
#### What if I want to index vectors with more than 2,000 dimensions?
|
||||
|
||||
Two things you can try are:
|
||||
You’ll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
|
||||
|
||||
1. use dimensionality reduction
|
||||
2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
|
||||
#### Why am I seeing less results after adding an index?
|
||||
|
||||
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
||||
|
||||
## Reference
|
||||
|
||||
@@ -260,6 +345,46 @@ Function | Description
|
||||
--- | ---
|
||||
avg(vector) → vector | arithmetic mean
|
||||
|
||||
## Installation Notes
|
||||
|
||||
### Postgres Location
|
||||
|
||||
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
|
||||
|
||||
```sh
|
||||
export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
|
||||
```
|
||||
|
||||
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
|
||||
|
||||
```sh
|
||||
sudo --preserve-env=PG_CONFIG make install
|
||||
```
|
||||
|
||||
### Missing Header
|
||||
|
||||
If compilation fails with `fatal error: postgres.h: No such file or directory`, make sure Postgres development files are installed on the server.
|
||||
|
||||
For Ubuntu and Debian, use:
|
||||
|
||||
```sh
|
||||
sudo apt-get install postgresql-server-dev-15
|
||||
```
|
||||
|
||||
Note: Replace `15` with your Postgres server version
|
||||
|
||||
### Windows
|
||||
|
||||
Support for Windows is currently experimental. Use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\15"
|
||||
git clone --branch v0.4.2 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
```
|
||||
|
||||
## Additional Installation Methods
|
||||
|
||||
### Docker
|
||||
@@ -275,7 +400,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.2 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build -t pgvector .
|
||||
```
|
||||
@@ -288,6 +413,8 @@ With Homebrew Postgres, you can use:
|
||||
brew install pgvector
|
||||
```
|
||||
|
||||
Note: This only adds it to the `postgresql@14` formula
|
||||
|
||||
### PGXN
|
||||
|
||||
Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) with:
|
||||
@@ -296,6 +423,18 @@ Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) wi
|
||||
pgxn install vector
|
||||
```
|
||||
|
||||
### Yum
|
||||
|
||||
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
|
||||
|
||||
```sh
|
||||
sudo yum install pgvector_15
|
||||
# or
|
||||
sudo dnf install pgvector_15
|
||||
```
|
||||
|
||||
Note: Replace `15` with your Postgres server version
|
||||
|
||||
### conda-forge
|
||||
|
||||
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
|
||||
@@ -312,11 +451,10 @@ pgvector is available on [these providers](https://github.com/pgvector/pgvector/
|
||||
|
||||
To request a new extension on other providers:
|
||||
|
||||
- Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
|
||||
- Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
|
||||
- Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
|
||||
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
|
||||
- Render - vote or comment on [this page](https://feedback.render.com/features/p/add-pgvector-extension-to-postgresql)
|
||||
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/managed-database/p/pgvector-extension-for-postgresql)
|
||||
- Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
|
||||
|
||||
## Upgrading
|
||||
|
||||
|
||||
2
sql/vector--0.4.1--0.4.2.sql
Normal file
2
sql/vector--0.4.1--0.4.2.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.2'" to load this file. \quit
|
||||
@@ -431,8 +431,18 @@ ComputeCenters(IvfflatBuildState * buildstate)
|
||||
/* TODO Ensure within maintenance_work_mem */
|
||||
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions);
|
||||
if (buildstate->heap != NULL)
|
||||
{
|
||||
SampleRows(buildstate);
|
||||
|
||||
if (buildstate->samples->length < buildstate->lists)
|
||||
{
|
||||
ereport(NOTICE,
|
||||
(errmsg("ivfflat index created with little data"),
|
||||
errdetail("This will cause low recall."),
|
||||
errhint("Drop the index until the table has more data.")));
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate centers */
|
||||
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
|
||||
|
||||
|
||||
@@ -111,6 +111,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
Datum datum;
|
||||
bool isnull;
|
||||
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
|
||||
double tuples = 0;
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
|
||||
@@ -159,6 +160,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
ExecStoreVirtualTuple(slot);
|
||||
|
||||
tuplesort_puttupleslot(so->sortstate, slot);
|
||||
|
||||
tuples++;
|
||||
}
|
||||
|
||||
searchPage = IvfflatPageGetOpaque(page)->nextblkno;
|
||||
@@ -167,6 +170,13 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO Scan more lists */
|
||||
if (tuples < 100)
|
||||
ereport(DEBUG1,
|
||||
(errmsg("index scan found few tuples"),
|
||||
errdetail("Index may have been created with little data."),
|
||||
errhint("Recreate the index and possibly decrease lists.")));
|
||||
|
||||
tuplesort_performsort(so->sortstate);
|
||||
}
|
||||
|
||||
|
||||
20
src/vector.c
20
src/vector.c
@@ -100,7 +100,7 @@ CheckStateArray(ArrayType *statearray, const char *caller)
|
||||
return (float8 *) ARR_DATA_PTR(statearray);
|
||||
}
|
||||
|
||||
#if PG_VERSION_NUM < 120000
|
||||
#if PG_VERSION_NUM < 120003
|
||||
static pg_noinline void
|
||||
float_overflow_error(void)
|
||||
{
|
||||
@@ -396,10 +396,8 @@ array_to_vector(PG_FUNCTION_ARGS)
|
||||
get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
|
||||
deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp);
|
||||
|
||||
if (typmod == -1)
|
||||
CheckDim(nelemsp);
|
||||
else
|
||||
CheckExpectedDim(typmod, nelemsp);
|
||||
CheckDim(nelemsp);
|
||||
CheckExpectedDim(typmod, nelemsp);
|
||||
|
||||
result = InitVector(nelemsp);
|
||||
for (i = 0; i < nelemsp; i++)
|
||||
@@ -467,6 +465,7 @@ l2_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
diff = ax[i] - bx[i];
|
||||
@@ -493,6 +492,7 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
diff = ax[i] - bx[i];
|
||||
@@ -517,6 +517,7 @@ inner_product(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += ax[i] * bx[i];
|
||||
|
||||
@@ -538,6 +539,7 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += ax[i] * bx[i];
|
||||
|
||||
@@ -561,6 +563,7 @@ cosine_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
{
|
||||
distance += ax[i] * bx[i];
|
||||
@@ -587,6 +590,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += a->x[i] * b->x[i];
|
||||
|
||||
@@ -622,6 +626,7 @@ vector_norm(PG_FUNCTION_ARGS)
|
||||
float *ax = a->x;
|
||||
double norm = 0.0;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
norm += ax[i] * ax[i];
|
||||
|
||||
@@ -646,6 +651,8 @@ vector_add(PG_FUNCTION_ARGS)
|
||||
|
||||
result = InitVector(a->dim);
|
||||
rx = result->x;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] + bx[i];
|
||||
|
||||
@@ -670,6 +677,8 @@ vector_sub(PG_FUNCTION_ARGS)
|
||||
|
||||
result = InitVector(a->dim);
|
||||
rx = result->x;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] - bx[i];
|
||||
|
||||
@@ -941,6 +950,7 @@ vector_avg(PG_FUNCTION_ARGS)
|
||||
|
||||
/* Create vector */
|
||||
dim = STATE_DIMS(statearray);
|
||||
CheckDim(dim);
|
||||
result = InitVector(dim);
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
|
||||
@@ -46,6 +46,8 @@ SELECT '[1,2,3]'::vector::real[];
|
||||
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
-- ensure no error
|
||||
SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
|
||||
?column?
|
||||
|
||||
@@ -102,3 +102,5 @@ SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
|
||||
ERROR: expected 2 dimensions, not 1
|
||||
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
|
||||
@@ -10,6 +10,7 @@ SELECT '{-Infinity}'::real[]::vector;
|
||||
SELECT '{}'::real[]::vector;
|
||||
SELECT '[1,2,3]'::vector::real[];
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
|
||||
-- ensure no error
|
||||
SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
|
||||
|
||||
@@ -24,3 +24,4 @@ SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
|
||||
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat access method'
|
||||
default_version = '0.4.1'
|
||||
default_version = '0.4.2'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user