Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
c8e4bf1c3c Added test for arrays 2024-02-03 16:50:17 -08:00
81 changed files with 395 additions and 4971 deletions

View File

@@ -1,14 +1,100 @@
name: build
on: [push, pull_request]
jobs:
valgrind:
ubuntu:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- postgres: 17
os: ubuntu-22.04
- postgres: 16
os: ubuntu-22.04
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-22.04
- postgres: 13
os: ubuntu-20.04
- postgres: 12
os: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1
- uses: ankane/setup-postgres@v1
with:
postgres-version: 16
- run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- run: |
sudo apt-get update
sudo apt-get install libipc-run-perl
- run: make prove_installcheck
mac:
runs-on: macos-latest
if: ${{ !startsWith(github.ref_name, 'windows') }}
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: make
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- run: |
brew install cpanm
cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
tar xf REL_14_10.tar.gz
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make PG_CFLAGS="-DUSE_ASSERT_CHECKING"
windows:
runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }}
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
i386:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
container:
image: debian:12
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare

View File

@@ -1,23 +1,6 @@
## 0.7.0 (unreleased)
- Added `halfvec` type
- Added `sparsevec` type
- Added support for `bit` vectors to HNSW
- Added `hamming_distance` function
- Added `jaccard_distance` function
- Added `quantize_binary` function
- Added `subvector` function
- Updated comparison operators to support vectors with different dimensions
## 0.6.2 (2024-03-18)
- Reduced lock contention with parallel HNSW index builds
## 0.6.1 (2024-03-04)
## 0.6.1 (unreleased)
- Fixed error with `ANALYZE` and vectors with different dimensions
- Fixed segmentation fault with `shared_preload_libraries`
- Fixed vector subtraction being marked as commutative
## 0.6.0 (2024-01-29)

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.6.2",
"version": "0.6.0",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.6.2",
"version": "0.6.0",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,10 +1,10 @@
EXTENSION = vector
EXTVERSION = 0.6.2
EXTVERSION = 0.6.0
MODULE_big = vector
DATA = $(wildcard sql/*--*.sql)
OBJS = src/bitvector.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
OBJS = src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/vector.o
HEADERS = src/vector.h
TESTS = $(wildcard test/sql/*.sql)
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
@@ -12,7 +12,7 @@ REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
OPTFLAGS = -march=native
# Mac ARM doesn't always support -march=native
# Mac ARM doesn't support -march=native
ifeq ($(shell uname -s), Darwin)
ifeq ($(shell uname -p), arm)
# no difference with -march=armv8.5-a
@@ -71,9 +71,11 @@ PG_MAJOR ?= 16
.PHONY: docker
docker:
docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
docker build --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
docker buildx build --push --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

View File

@@ -1,10 +1,10 @@
EXTENSION = vector
EXTVERSION = 0.6.2
EXTVERSION = 0.6.0
OBJS = src\bitvector.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
HEADERS = src\vector.h
REGRESS = bit_functions btree cast copy halfvec_functions halfvec_input hnsw_bit_hamming hnsw_bit_jaccard hnsw_halfvec_cosine hnsw_halfvec_ip hnsw_halfvec_l2 hnsw_options hnsw_sparsevec_cosine hnsw_sparsevec_ip hnsw_sparsevec_l2 hnsw_unlogged hnsw_vector_cosine hnsw_vector_ip hnsw_vector_l2 ivfflat_options ivfflat_unlogged ivfflat_vector_cosine ivfflat_vector_ip ivfflat_vector_l2 sparsevec_functions sparsevec_input vector_functions vector_input
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# For /arch flags

364
README.md
View File

@@ -20,15 +20,15 @@ Compile and install the extension (supports Postgres 12+)
```sh
cd /tmp
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
```
See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
See the [installation notes](#installation-notes) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
### Windows
@@ -44,15 +44,12 @@ Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP%
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
```
See the [installation notes](#installation-notes---windows) if you run into issues
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
## Getting Started
@@ -105,12 +102,6 @@ Insert vectors
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
```
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py))
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
```
Upsert vectors
```sql
@@ -221,24 +212,7 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
```
Hamming distance - unreleased
```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
```
Jaccard distance - unreleased
```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
```
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions (unreleased)
- `bit` - up to 64,000 dimensions (unreleased)
- `sparsevec` - up to 1,000 non-zero elements (unreleased)
Vectors with up to 2,000 dimensions can be indexed.
### Index Options
@@ -290,8 +264,6 @@ HINT: Increase maintenance_work_mem to speed up builds.
Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on the server
Like other index types, its faster to create an index after loading your initial data
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql
@@ -343,9 +315,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
```
Supported types are:
- `vector` - up to 2,000 dimensions
Vectors with up to 2,000 dimensions can be indexed.
### Query Options
@@ -419,103 +389,6 @@ Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
```
## Half Vectors
*Unreleased*
Use the `halfvec` type to store half-precision vectors
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
```
## Half Indexing
*Unreleased*
Index vectors at half precision for smaller indexes and faster build times
```sql
CREATE INDEX ON items USING hnsw ((embedding::halfvec(3)) halfvec_l2_ops);
```
Get the nearest neighbors
```sql
SELECT * FROM items ORDER BY embedding::halfvec(3) <-> '[1,2,3]' LIMIT 5;
```
## Binary Vectors
Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py))
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111');
```
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
```
Or (unreleased)
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Also supports Jaccard distance (`<%>`)
## Binary Quantization
*Unreleased*
Use expression indexing for binary quantization
```sql
CREATE INDEX ON items USING hnsw ((quantize_binary(embedding)::bit(3)) bit_hamming_ops);
```
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY quantize_binary(embedding)::bit(3) <~> quantize_binary('[1,-2,3]') LIMIT 5;
```
Re-rank by the original vectors for better recall
```sql
SELECT * FROM (
SELECT * FROM items ORDER BY quantize_binary(embedding)::bit(3) <~> quantize_binary('[1,-2,3]') LIMIT 20
) ORDER BY embedding <=> '[1,-2,3]' LIMIT 5;
```
## Sparse Vectors
*Unreleased*
Use the `sparsevec` type to store sparse vectors
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding sparsevec(5));
```
Insert vectors
```sql
INSERT INTO items (embedding) VALUES ('{1:1,3:2,5:3}/5'), ('{1:4,3:5,5:6}/5');
```
Note: The format is `{index1:value1,index2:value2,...}/dimensions` and indices start at 1 like SQL arrays
Get the nearest neighbors by L2 distance
```sql
SELECT * FROM items ORDER BY embedding <-> '{1:3,3:1,5:2}/5' LIMIT 5;
```
## Hybrid Search
Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search.
@@ -529,51 +402,13 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
## Performance
### Tuning
Use a tool like [PgTune](https://pgtune.leopard.in.ua/) to set initial values for Postgres server parameters. For instance, `shared_buffers` should typically be 25% of the servers memory. You can find the config file with:
```sql
SHOW config_file;
```
And check individual settings with:
```sql
SHOW shared_buffers;
```
Be sure to restart Postgres for changes to take effect.
### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py)).
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
```
Add any indexes *after* loading the initial data for best performance.
### Indexing
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
In production environments, create indexes concurrently to avoid blocking writes.
```sql
CREATE INDEX CONCURRENTLY ...
```
### Querying
Use `EXPLAIN ANALYZE` to debug performance.
```sql
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
#### Exact Search
### Exact Search
To speed up queries without an index, increase `max_parallel_workers_per_gather`.
@@ -587,7 +422,7 @@ If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
```
#### Approximate Search
### Approximate Search
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
@@ -595,7 +430,7 @@ To speed up queries with an IVFFlat index, increase the number of inverted lists
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
```
### Vacuuming
## Vacuuming
Vacuuming can take a while for HNSW indexes. Speed it up by reindexing first.
@@ -604,41 +439,6 @@ REINDEX INDEX CONCURRENTLY index_name;
VACUUM table_name;
```
## Monitoring
Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
```sql
CREATE EXTENSION pg_stat_statements;
```
Get the most time-consuming queries with:
```sql
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
Monitor recall by comparing results from approximate search with exact search.
```sql
BEGIN;
SET LOCAL enable_indexscan = off; -- use exact search
SELECT ...
COMMIT;
```
## Scaling
Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
@@ -744,17 +544,7 @@ SELECT pg_size_pretty(pg_relation_size('index_name'));
#### Why isnt a query using an index?
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator, not an expression.
```sql
-- index
ORDER BY embedding <=> '[3,1,2]' LIMIT 5;
-- no index
ORDER BY 1 - (embedding <=> '[3,1,2]') DESC LIMIT 5;
```
You can encourage the planner to use an index for a query with:
The cost estimation in pgvector < 0.4.3 does not always work well with the planner. You can encourage the planner to use an index for a query with:
```sql
BEGIN;
@@ -783,12 +573,6 @@ or choose to store vectors inline:
ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
```
#### Why are there less results for a query after adding an HNSW index?
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
#### Why are there less results for a query after adding an IVFFlat index?
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
@@ -797,20 +581,11 @@ The index was likely created with too little data for the number of lists. Drop
DROP INDEX index_name;
```
Results can also be limited by the number of probes (`ivfflat.probes`).
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
## Reference
- [Vector](#vector-type)
- [Halfvec](#halfvec-type)
- [Bit](#bit-type)
- [Sparsevec](#sparsevec-type)
### Vector Type
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single-precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
### Vector Operators
@@ -831,81 +606,17 @@ cosine_distance(vector, vector) → double precision | cosine distance |
inner_product(vector, vector) → double precision | inner product |
l2_distance(vector, vector) → double precision | Euclidean distance |
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
quantize_binary(vector) → bit | quantize | unreleased
subvector(vector, integer, integer) → vector | subvector | unreleased
vector_dims(vector) → integer | number of dimensions |
vector_norm(vector) → double precision | Euclidean norm |
### Vector Aggregate Functions
### Aggregate Functions
Function | Description | Added
--- | --- | ---
avg(vector) → vector | average |
sum(vector) → vector | sum | 0.5.0
### Halfvec Type
Each half vector takes `2 * dimensions + 8` bytes of storage. Each element is a half-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Half vectors can have up to 16,000 dimensions.
### Halfvec Operators
Operator | Description | Added
--- | --- | ---
<-> | Euclidean distance | unreleased
<#> | negative inner product | unreleased
<=> | cosine distance | unreleased
### Halfvec Functions
Function | Description | Added
--- | --- | ---
cosine_distance(halfvec, halfvec) → double precision | cosine distance | unreleased
inner_product(halfvec, halfvec) → double precision | inner product | unreleased
l2_distance(halfvec, halfvec) → double precision | Euclidean distance | unreleased
l1_distance(halfvec, halfvec) → double precision | taxicab distance | unreleased
quantize_binary(halfvec) → bit | quantize | unreleased
subvector(halfvec, integer, integer) → halfvec | subvector | unreleased
### Bit Type
Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres docs](https://www.postgresql.org/docs/current/datatype-bit.html) for more info.
### Bit Operators
Operator | Description | Added
--- | --- | ---
<~> | Hamming distance | unreleased
<%> | Jaccard distance | unreleased
### Bit Functions
Function | Description | Added
--- | --- | ---
hamming_distance(bit, bit) → double precision | Hamming distance | unreleased
jaccard_distance(bit, bit) → double precision | Jaccard distance | unreleased
### Sparsevec Type
Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each element is a single-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Sparse vectors can have up to 16,000 non-zero elements.
### Sparsevec Operators
Operator | Description | Added
--- | --- | ---
<-> | Euclidean distance | unreleased
<#> | negative inner product | unreleased
<=> | cosine distance | unreleased
### Sparsevec Functions
Function | Description | Added
--- | --- | ---
cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | unreleased
inner_product(sparsevec, sparsevec) → double precision | inner product | unreleased
l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | unreleased
l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | unreleased
## Installation Notes - Linux and Mac
## Installation Notes
### Postgres Location
@@ -945,26 +656,6 @@ Note: Replace `16` with your Postgres server version
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
### Portability
By default, pgvector compiles with `-march=native` on some platforms for best performance. However, this can lead to `Illegal instruction` errors if trying to run the compiled extension on a different machine.
To compile for portability, use:
```sh
make OPTFLAGS=""
```
## Installation Notes - Windows
### Missing Header
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
### Permissions
If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
## Additional Installation Methods
### Docker
@@ -980,7 +671,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually:
```sh
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
```
@@ -1025,21 +716,6 @@ sudo dnf install pgvector_16
Note: Replace `16` with your Postgres server version
### pkg
Install the FreeBSD package with:
```sh
pkg install postgresql15-pg_vector
```
or the port with:
```sh
cd /usr/ports/databases/pgvector
make install
```
### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
@@ -1149,12 +825,6 @@ make installcheck REGRESS=functions # regression test
make prove_installcheck PROVE_TESTS=test/t/001_ivfflat_wal.pl # TAP test
```
To enable assertions:
```sh
make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
```
To enable benchmarking:
```sh
@@ -1167,6 +837,12 @@ To show memory usage:
make clean && PG_CFLAGS="-DHNSW_MEMORY -DIVFFLAT_MEMORY" make && make install
```
To enable assertions:
```sh
make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
```
To get k-means metrics:
```sh

View File

@@ -1,16 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.6.1'" to load this file. \quit
DROP OPERATOR - (vector, vector);
CREATE OPERATOR - (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_sub
);
ALTER OPERATOR <= (vector, vector) SET (
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
ALTER OPERATOR >= (vector, vector) SET (
RESTRICT = scalargesel, JOIN = scalargejoinsel
);

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.6.2'" to load this file. \quit

View File

@@ -1,259 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.0'" to load this file. \quit
CREATE FUNCTION quantize_binary(vector) RETURNS bit
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit);
CREATE TYPE halfvec;
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_out(halfvec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_recv(internal, oid, integer) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_send(halfvec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE halfvec (
INPUT = halfvec_in,
OUTPUT = halfvec_out,
TYPMOD_IN = halfvec_typmod_in,
RECEIVE = halfvec_recv,
SEND = halfvec_send,
STORAGE = external
);
CREATE FUNCTION l2_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_norm(halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION quantize_binary(halfvec) RETURNS bit
AS 'MODULE_PATHNAME', 'halfvec_quantize_binary' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(halfvec, int, int) RETURNS halfvec
AS 'MODULE_PATHNAME', 'halfvec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_l2_squared_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_negative_inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec(halfvec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(integer[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(real[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(double precision[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(numeric[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_float4(halfvec, integer, boolean) RETURNS real[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (halfvec AS halfvec)
WITH FUNCTION halfvec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (halfvec AS real[])
WITH FUNCTION halfvec_to_float4(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (integer[] AS halfvec)
WITH FUNCTION array_to_halfvec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS halfvec)
WITH FUNCTION array_to_halfvec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS halfvec)
WITH FUNCTION array_to_halfvec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS halfvec)
WITH FUNCTION array_to_halfvec(numeric[], integer, boolean) AS ASSIGNMENT;
CREATE OPERATOR <-> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 halfvec_norm(halfvec);
CREATE FUNCTION halfvec_to_vector(halfvec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_halfvec(vector, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (halfvec AS vector)
WITH FUNCTION halfvec_to_vector(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (vector AS halfvec)
WITH FUNCTION vector_to_halfvec(vector, integer, boolean) AS IMPLICIT;
CREATE TYPE sparsevec;
CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_send(sparsevec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE sparsevec (
INPUT = sparsevec_in,
OUTPUT = sparsevec_out,
TYPMOD_IN = sparsevec_typmod_in,
RECEIVE = sparsevec_recv,
SEND = sparsevec_send,
STORAGE = external
);
CREATE FUNCTION l2_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_norm(sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_l2_squared_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_negative_inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec(sparsevec, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_sparsevec(vector, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (sparsevec AS sparsevec)
WITH FUNCTION sparsevec(sparsevec, integer, boolean) AS IMPLICIT;
CREATE CAST (sparsevec AS vector)
WITH FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) AS IMPLICIT;
CREATE CAST (vector AS sparsevec)
WITH FUNCTION vector_to_sparsevec(vector, integer, boolean) AS IMPLICIT;
CREATE OPERATOR <-> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
CREATE OPERATOR CLASS sparsevec_l2_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec);
CREATE OPERATOR CLASS sparsevec_ip_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec);
CREATE OPERATOR CLASS sparsevec_cosine_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 2 sparsevec_norm(sparsevec);

View File

@@ -1,7 +1,7 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION vector" to load this file. \quit
-- vector type
-- type
CREATE TYPE vector;
@@ -29,7 +29,7 @@ CREATE TYPE vector (
STORAGE = external
);
-- vector functions
-- functions
CREATE FUNCTION l2_distance(vector, vector) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -58,13 +58,7 @@ CREATE FUNCTION vector_sub(vector, vector) RETURNS vector
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION quantize_binary(vector) RETURNS bit
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector private functions
-- private functions
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -105,7 +99,7 @@ CREATE FUNCTION vector_avg(double precision[]) RETURNS vector
CREATE FUNCTION vector_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector aggregates
-- aggregates
CREATE AGGREGATE avg(vector) (
SFUNC = vector_accum,
@@ -123,7 +117,7 @@ CREATE AGGREGATE sum(vector) (
PARALLEL = SAFE
);
-- vector cast functions
-- cast functions
CREATE FUNCTION vector(vector, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -143,7 +137,7 @@ CREATE FUNCTION array_to_vector(numeric[], integer, boolean) RETURNS vector
CREATE FUNCTION vector_to_float4(vector, integer, boolean) RETURNS real[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector casts
-- casts
CREATE CAST (vector AS vector)
WITH FUNCTION vector(vector, integer, boolean) AS IMPLICIT;
@@ -163,7 +157,7 @@ CREATE CAST (double precision[] AS vector)
CREATE CAST (numeric[] AS vector)
WITH FUNCTION array_to_vector(numeric[], integer, boolean) AS ASSIGNMENT;
-- vector operators
-- operators
CREATE OPERATOR <-> (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l2_distance,
@@ -186,7 +180,8 @@ CREATE OPERATOR + (
);
CREATE OPERATOR - (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_sub
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_sub,
COMMUTATOR = -
);
CREATE OPERATOR * (
@@ -200,10 +195,11 @@ CREATE OPERATOR < (
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
-- should use scalarlesel and scalarlejoinsel, but not supported in Postgres < 11
CREATE OPERATOR <= (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_le,
COMMUTATOR = >= , NEGATOR = > ,
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
CREATE OPERATOR = (
@@ -218,10 +214,11 @@ CREATE OPERATOR <> (
RESTRICT = eqsel, JOIN = eqjoinsel
);
-- should use scalargesel and scalargejoinsel, but not supported in Postgres < 11
CREATE OPERATOR >= (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_ge,
COMMUTATOR = <= , NEGATOR = < ,
RESTRICT = scalargesel, JOIN = scalargejoinsel
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
);
CREATE OPERATOR > (
@@ -246,7 +243,7 @@ CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnswhandler;
COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
-- vector opclasses
-- opclasses
CREATE OPERATOR CLASS vector_ops
DEFAULT FOR TYPE vector USING btree AS
@@ -293,286 +290,3 @@ CREATE OPERATOR CLASS vector_cosine_ops
OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 vector_negative_inner_product(vector, vector),
FUNCTION 2 vector_norm(vector);
-- bit functions
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit);
-- halfvec type
CREATE TYPE halfvec;
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_out(halfvec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_recv(internal, oid, integer) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_send(halfvec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE halfvec (
INPUT = halfvec_in,
OUTPUT = halfvec_out,
TYPMOD_IN = halfvec_typmod_in,
RECEIVE = halfvec_recv,
SEND = halfvec_send,
STORAGE = external
);
-- halfvec functions
CREATE FUNCTION l2_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_norm(halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION quantize_binary(halfvec) RETURNS bit
AS 'MODULE_PATHNAME', 'halfvec_quantize_binary' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(halfvec, int, int) RETURNS halfvec
AS 'MODULE_PATHNAME', 'halfvec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec private functions
CREATE FUNCTION halfvec_l2_squared_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_negative_inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec cast functions
CREATE FUNCTION halfvec(halfvec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(integer[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(real[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(double precision[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(numeric[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_float4(halfvec, integer, boolean) RETURNS real[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec casts
CREATE CAST (halfvec AS halfvec)
WITH FUNCTION halfvec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (halfvec AS real[])
WITH FUNCTION halfvec_to_float4(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (integer[] AS halfvec)
WITH FUNCTION array_to_halfvec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS halfvec)
WITH FUNCTION array_to_halfvec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS halfvec)
WITH FUNCTION array_to_halfvec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS halfvec)
WITH FUNCTION array_to_halfvec(numeric[], integer, boolean) AS ASSIGNMENT;
-- halfvec operators
CREATE OPERATOR <-> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
-- halfvec opclasses
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 halfvec_norm(halfvec);
-- extension casts
CREATE FUNCTION halfvec_to_vector(halfvec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_halfvec(vector, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (halfvec AS vector)
WITH FUNCTION halfvec_to_vector(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (vector AS halfvec)
WITH FUNCTION vector_to_halfvec(vector, integer, boolean) AS IMPLICIT;
--- sparsevec type
CREATE TYPE sparsevec;
CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_send(sparsevec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE sparsevec (
INPUT = sparsevec_in,
OUTPUT = sparsevec_out,
TYPMOD_IN = sparsevec_typmod_in,
RECEIVE = sparsevec_recv,
SEND = sparsevec_send,
STORAGE = external
);
-- sparsevec functions
CREATE FUNCTION l2_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_norm(sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec private functions
CREATE FUNCTION sparsevec_l2_squared_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_negative_inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec cast functions
CREATE FUNCTION sparsevec(sparsevec, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_sparsevec(vector, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec casts
CREATE CAST (sparsevec AS sparsevec)
WITH FUNCTION sparsevec(sparsevec, integer, boolean) AS IMPLICIT;
CREATE CAST (sparsevec AS vector)
WITH FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) AS IMPLICIT;
CREATE CAST (vector AS sparsevec)
WITH FUNCTION vector_to_sparsevec(vector, integer, boolean) AS IMPLICIT;
-- sparsevec operators
CREATE OPERATOR <-> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
-- sparsevec opclasses
CREATE OPERATOR CLASS sparsevec_l2_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec);
CREATE OPERATOR CLASS sparsevec_ip_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec);
CREATE OPERATOR CLASS sparsevec_cosine_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 2 sparsevec_norm(sparsevec);

View File

@@ -1,90 +0,0 @@
#include "postgres.h"
#include "bitvector.h"
#include "port/pg_bitutils.h"
#include "utils/varbit.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate and initialize a new bit vector
*/
VarBit *
InitBitVector(int dim)
{
VarBit *result;
int size;
size = VARBITTOTALLEN(dim);
result = (VarBit *) palloc0(size);
SET_VARSIZE(result, size);
VARBITLEN(result) = dim;
return result;
}
/*
* Ensure same dimensions
*/
static inline void
CheckDims(VarBit *a, VarBit *b)
{
if (VARBITLEN(a) != VARBITLEN(b))
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("different bit lengths %u and %u", VARBITLEN(a), VARBITLEN(b))));
}
/*
* Get the Hamming distance between two bit vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
Datum
hamming_distance(PG_FUNCTION_ARGS)
{
VarBit *a = PG_GETARG_VARBIT_P(0);
VarBit *b = PG_GETARG_VARBIT_P(1);
unsigned char *ax = VARBITS(a);
unsigned char *bx = VARBITS(b);
uint64 distance = 0;
CheckDims(a, b);
/* TODO Improve performance */
for (uint32 i = 0; i < VARBITBYTES(a); i++)
distance += pg_number_of_ones[ax[i] ^ bx[i]];
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the Jaccard distance between two bit vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(jaccard_distance);
Datum
jaccard_distance(PG_FUNCTION_ARGS)
{
VarBit *a = PG_GETARG_VARBIT_P(0);
VarBit *b = PG_GETARG_VARBIT_P(1);
unsigned char *ax = VARBITS(a);
unsigned char *bx = VARBITS(b);
uint64 ab = 0;
uint64 aa;
uint64 bb;
CheckDims(a, b);
/* TODO Improve performance */
for (uint32 i = 0; i < VARBITBYTES(a); i++)
ab += pg_number_of_ones[ax[i] & bx[i]];
if (ab == 0)
PG_RETURN_FLOAT8(1);
aa = pg_popcount((char *) ax, VARBITBYTES(a));
bb = pg_popcount((char *) bx, VARBITBYTES(b));
PG_RETURN_FLOAT8(1 - (ab / ((double) (aa + bb - ab))));
}

View File

@@ -1,8 +0,0 @@
#ifndef BITVECTOR_H
#define BITVECTOR_H
#include "utils/varbit.h"
VarBit *InitBitVector(int dim);
#endif

File diff suppressed because it is too large Load Diff

View File

@@ -1,45 +0,0 @@
#ifndef HALFVEC_H
#define HALFVEC_H
#define __STDC_WANT_IEC_60559_TYPES_EXT__
#include <float.h>
#include "vector.h"
#if defined(__F16C__)
#define F16C_SUPPORT
#elif defined(__FLT16_MAX__)
#define FLT16_SUPPORT
#endif
#ifdef FLT16_SUPPORT
#define half _Float16
#define HALF_MAX FLT16_MAX
#else
/* TODO #pragma message("")? */
#define half uint16
#define HALF_MAX 65504
#endif
#define HALFVEC_MAX_DIM VECTOR_MAX_DIM
#define HALFVEC_SIZE(_dim) (offsetof(HalfVector, x) + sizeof(half)*(_dim))
#define DatumGetHalfVector(x) ((HalfVector *) PG_DETOAST_DATUM(x))
#define PG_GETARG_HALFVEC_P(x) DatumGetHalfVector(PG_GETARG_DATUM(x))
#define PG_RETURN_HALFVEC_P(x) PG_RETURN_POINTER(x)
typedef struct HalfVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int16 dim; /* number of dimensions */
int16 unused;
half x[FLEXIBLE_ARRAY_MEMBER];
} HalfVector;
HalfVector *InitHalfVector(int dim);
float HalfToFloat4(half num);
half Float4ToHalf(float num);
half Float4ToHalfUnchecked(float num);
#endif

View File

@@ -8,7 +8,6 @@
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "utils/guc.h"
#include "utils/selfuncs.h"
@@ -29,7 +28,7 @@ static relopt_kind hnsw_relopt_kind;
* this grows bigger, we should use a shmem_request_hook and
* RequestAddinShmemSpace() to pre-reserve space for this.
*/
void
static void
HnswInitLockTranche(void)
{
int *tranche_ids;
@@ -54,8 +53,7 @@ HnswInitLockTranche(void)
void
HnswInit(void)
{
if (!process_shared_preload_libraries_in_progress)
HnswInitLockTranche();
HnswInitLockTranche();
hnsw_relopt_kind = add_reloption_kind();
add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",

View File

@@ -17,7 +17,6 @@
#endif
#define HNSW_MAX_DIM 2000
#define HNSW_MAX_NNZ 1000
/* Support functions */
#define HNSW_DISTANCE_PROC 1
@@ -56,14 +55,6 @@
#define HNSW_UPDATE_ENTRY_GREATER 1
#define HNSW_UPDATE_ENTRY_ALWAYS 2
typedef enum HnswType
{
HNSW_TYPE_VECTOR,
HNSW_TYPE_HALFVEC,
HNSW_TYPE_BIT,
HNSW_TYPE_SPARSEVEC
} HnswType;
/* Build phases */
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
#define PROGRESS_HNSW_PHASE_LOAD 2
@@ -89,7 +80,7 @@ typedef enum HnswType
#if PG_VERSION_NUM < 130000
#define list_delete_last(list) list_truncate(list, list_length(list) - 1)
#define list_sort(list, cmp) ((list) = list_qsort(list, cmp))
#define list_sort(list, cmp) list_qsort(list, cmp)
#endif
#define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
@@ -138,7 +129,7 @@ HnswPtrDeclare(HnswNeighborArray, HnswNeighborArrayRelptr, HnswNeighborArrayPtr)
HnswPtrDeclare(HnswNeighborArrayPtr, HnswNeighborsRelptr, HnswNeighborsPtr);
HnswPtrDeclare(char, DatumRelptr, DatumPtr);
struct HnswElementData
typedef struct HnswElementData
{
HnswElementPtr next;
ItemPointerData heaptids[HNSW_HEAPTIDS];
@@ -153,7 +144,7 @@ struct HnswElementData
BlockNumber neighborPage;
DatumPtr value;
LWLock lock;
};
} HnswElementData;
typedef HnswElementData * HnswElement;
@@ -164,12 +155,12 @@ typedef struct HnswCandidate
bool closer;
} HnswCandidate;
struct HnswNeighborArray
typedef struct HnswNeighborArray
{
int length;
bool closerSet;
HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
};
} HnswNeighborArray;
typedef struct HnswPairingHeapNode
{
@@ -194,7 +185,6 @@ typedef struct HnswGraph
/* Entry state */
LWLock entryLock;
LWLock entryWaitLock;
HnswElementPtr entryPoint;
/* Allocations state */
@@ -251,7 +241,6 @@ typedef struct HnswBuildState
Relation index;
IndexInfo *indexInfo;
ForkNumber forkNum;
HnswType type;
/* Settings */
int dimensions;
@@ -272,6 +261,7 @@ typedef struct HnswBuildState
HnswGraph *graph;
double ml;
int maxLevel;
Vector *normvec;
/* Memory */
MemoryContext graphCtx;
@@ -376,9 +366,7 @@ typedef struct HnswVacuumState
int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
HnswType HnswGetType(Relation index);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type);
void HnswCheckValue(Datum value, HnswType type);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page);
void HnswInit(void);
@@ -401,7 +389,6 @@ void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation i
void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
void HnswLoadNeighbors(HnswElement element, Relation index, int m);
void HnswInitLockTranche(void);
PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
/* Index access methods */

View File

@@ -44,7 +44,6 @@
#include "access/xact.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h"
#include "hnsw.h"
#include "miscadmin.h"
@@ -177,7 +176,7 @@ CreateGraphPages(HnswBuildState * buildstate)
Size etupSize;
Size ntupSize;
Size combinedSize;
Pointer valuePtr = HnswPtrAccess(base, element->value);
void *valuePtr = HnswPtrAccess(base, element->value);
/* Update iterator */
iter = element->next;
@@ -432,15 +431,10 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
HnswGraph *graph = buildstate->graph;
HnswElement entryPoint;
LWLock *entryLock = &graph->entryLock;
LWLock *entryWaitLock = &graph->entryWaitLock;
int efConstruction = buildstate->efConstruction;
int m = buildstate->m;
char *base = buildstate->hnswarea;
/* Wait if another process needs exclusive lock on entry lock */
LWLockAcquire(entryWaitLock, LW_EXCLUSIVE);
LWLockRelease(entryWaitLock);
/* Get entry point */
LWLockAcquire(entryLock, LW_SHARED);
entryPoint = HnswPtrAccess(base, graph->entryPoint);
@@ -451,10 +445,8 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
/* Release shared lock */
LWLockRelease(entryLock);
/* Tell other processes to wait and get exclusive lock */
LWLockAcquire(entryWaitLock, LW_EXCLUSIVE);
/* Get exclusive lock */
LWLockAcquire(entryLock, LW_EXCLUSIVE);
LWLockRelease(entryWaitLock);
/* Get latest entry point after lock is acquired */
entryPoint = HnswPtrAccess(base, graph->entryPoint);
@@ -487,13 +479,10 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Detoast once for all calls */
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Check value */
HnswCheckValue(value, buildstate->type);
/* Normalize if needed */
if (buildstate->normprocinfo != NULL)
{
if (!HnswNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->type))
if (!HnswNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->normvec))
return false;
}
@@ -612,9 +601,6 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
static void
InitGraph(HnswGraph * graph, char *base, long memoryTotal)
{
/* Initialize the lock tranche if needed */
HnswInitLockTranche();
HnswPtrStore(base, graph->head, (HnswElement) NULL);
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
graph->memoryUsed = 0;
@@ -623,7 +609,6 @@ InitGraph(HnswGraph * graph, char *base, long memoryTotal)
graph->indtuples = 0;
SpinLockInit(&graph->lock);
LWLockInitialize(&graph->entryLock, hnsw_lock_tranche_id);
LWLockInitialize(&graph->entryWaitLock, hnsw_lock_tranche_id);
LWLockInitialize(&graph->allocatorLock, hnsw_lock_tranche_id);
LWLockInitialize(&graph->flushLock, hnsw_lock_tranche_id);
}
@@ -669,50 +654,23 @@ HnswSharedMemoryAlloc(Size size, void *state)
return chunk;
}
/*
* Get max dimensions
*/
static int
GetMaxDimensions(HnswType type)
{
int maxDimensions = HNSW_MAX_DIM;
if (type == HNSW_TYPE_HALFVEC)
maxDimensions *= 2;
else if (type == HNSW_TYPE_BIT)
maxDimensions *= 32;
else if (type == HNSW_TYPE_SPARSEVEC)
maxDimensions = INT_MAX;
return maxDimensions;
}
/*
* Initialize the build state
*/
static void
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
{
int maxDimensions;
buildstate->heap = heap;
buildstate->index = index;
buildstate->indexInfo = indexInfo;
buildstate->forkNum = forkNum;
buildstate->type = HnswGetType(index);
buildstate->m = HnswGetM(index);
buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
maxDimensions = GetMaxDimensions(buildstate->type);
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > maxDimensions)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", maxDimensions);
if (buildstate->dimensions > HNSW_MAX_DIM)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", HNSW_MAX_DIM);
if (buildstate->efConstruction < 2 * buildstate->m)
elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
@@ -730,6 +688,9 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
/* Reuse for each tuple */
buildstate->normvec = InitVector(buildstate->dimensions);
buildstate->graphCtx = GenerationContextCreate(CurrentMemoryContext,
"Hnsw build graph context",
#if PG_VERSION_NUM >= 150000
@@ -753,6 +714,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
static void
FreeBuildState(HnswBuildState * buildstate)
{
pfree(buildstate->normvec);
MemoryContextDelete(buildstate->graphCtx);
MemoryContextDelete(buildstate->tmpCtx);
}
@@ -1008,14 +970,6 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Report less than allocated so never fails */
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
/*
* Avoid base address for relptr for Postgres < 14.5
* https://github.com/postgres/postgres/commit/7201cd18627afc64850537806da7f22150d1a83b
*/
#if PG_VERSION_NUM < 140005
hnswshared->graphData.memoryUsed += MAXALIGN(1);
#endif
shm_toc_insert(pcxt->toc, PARALLEL_KEY_HNSW_SHARED, hnswshared);
shm_toc_insert(pcxt->toc, PARALLEL_KEY_HNSW_AREA, hnswarea);

View File

@@ -614,19 +614,15 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
Datum value;
FmgrInfo *normprocinfo;
Oid collation = index->rd_indcollation[0];
HnswType type = HnswGetType(index);
/* Detoast once for all calls */
value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Check value */
HnswCheckValue(value, type);
/* Normalize if needed */
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
if (normprocinfo != NULL)
{
if (!HnswNormValue(normprocinfo, collation, &value, type))
if (!HnswNormValue(normprocinfo, collation, &value, NULL))
return;
}

View File

@@ -40,6 +40,29 @@ GetScanItems(IndexScanDesc scan, Datum q)
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
}
/*
* Get dimensions from metapage
*/
static int
GetDimensions(Relation index)
{
Buffer buf;
Page page;
HnswMetaPage metap;
int dimensions;
buf = ReadBuffer(index, HNSW_METAPAGE_BLKNO);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
metap = HnswPageGetMeta(page);
dimensions = metap->dimensions;
UnlockReleaseBuffer(buf);
return dimensions;
}
/*
* Get scan value
*/
@@ -50,7 +73,7 @@ GetScanValue(IndexScanDesc scan)
Datum value;
if (scan->orderByData->sk_flags & SK_ISNULL)
value = PointerGetDatum(NULL);
value = PointerGetDatum(InitVector(GetDimensions(scan->indexRelation)));
else
{
value = scan->orderByData->sk_argument;
@@ -61,7 +84,7 @@ GetScanValue(IndexScanDesc scan)
/* Fine if normalization fails */
if (so->normprocinfo != NULL)
HnswNormValue(so->normprocinfo, so->collation, &value, HnswGetType(scan->indexRelation));
HnswNormValue(so->normprocinfo, so->collation, &value, NULL);
}
return value;
@@ -158,10 +181,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
so->first = false;
#if defined(HNSW_MEMORY) && PG_VERSION_NUM >= 130000
elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(so->tmpCtx, false) / (1024 * 1024));
#endif
}
while (list_length(so->w) > 0)

View File

@@ -3,17 +3,12 @@
#include <math.h>
#include "access/generic_xlog.h"
#include "catalog/pg_type.h"
#include "catalog/pg_type_d.h"
#include "halfvec.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "sparsevec.h"
#include "storage/bufmgr.h"
#include "utils/datum.h"
#include "utils/memdebug.h"
#include "utils/rel.h"
#include "utils/syscache.h"
#include "vector.h"
#if PG_VERSION_NUM >= 130000
@@ -154,42 +149,6 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
return index_getprocinfo(index, 1, procnum);
}
/*
* Get type
*/
HnswType
HnswGetType(Relation index)
{
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
HeapTuple tuple;
Form_pg_type type;
HnswType result;
if (typid == BITOID)
return HNSW_TYPE_BIT;
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid));
if (!HeapTupleIsValid(tuple))
elog(ERROR, "cache lookup failed for type %u", typid);
type = (Form_pg_type) GETSTRUCT(tuple);
if (strcmp(NameStr(type->typname), "vector") == 0)
result = HNSW_TYPE_VECTOR;
else if (strcmp(NameStr(type->typname), "halfvec") == 0)
result = HNSW_TYPE_HALFVEC;
else if (strcmp(NameStr(type->typname), "sparsevec") == 0)
result = HNSW_TYPE_SPARSEVEC;
else
{
ReleaseSysCache(tuple);
elog(ERROR, "type not supported for hnsw index");
}
ReleaseSysCache(tuple);
return result;
}
/*
* Divide by the norm
*
@@ -199,53 +158,21 @@ HnswGetType(Relation index)
* if it's different than the original value
*/
bool
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type)
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result)
{
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
if (norm > 0)
{
/* TODO Remove vector-specific code */
if (type == HNSW_TYPE_VECTOR)
{
Vector *v = DatumGetVector(*value);
Vector *result = InitVector(v->dim);
Vector *v = DatumGetVector(*value);
for (int i = 0; i < v->dim; i++)
result->x[i] = v->x[i] / norm;
if (result == NULL)
result = InitVector(v->dim);
*value = PointerGetDatum(result);
}
else if (type == HNSW_TYPE_HALFVEC)
{
HalfVector *v = DatumGetHalfVector(*value);
HalfVector *result = InitHalfVector(v->dim);
for (int i = 0; i < v->dim; i++)
result->x[i] = v->x[i] / norm;
for (int i = 0; i < v->dim; i++)
{
/* TODO Fix */
result->x[i] = Float4ToHalfUnchecked(HalfToFloat4(v->x[i]) / norm);
}
*value = PointerGetDatum(result);
}
else if (type == HNSW_TYPE_SPARSEVEC)
{
SparseVector *v = DatumGetSparseVector(*value);
SparseVector *result = InitSparseVector(v->dim, v->nnz);
float *vx = SPARSEVEC_VALUES(v);
float *rx = SPARSEVEC_VALUES(result);
for (int i = 0; i < v->nnz; i++)
{
result->indices[i] = v->indices[i];
rx[i] = vx[i] / norm;
}
*value = PointerGetDatum(result);
}
else
elog(ERROR, "Unsupported type");
*value = PointerGetDatum(result);
return true;
}
@@ -253,21 +180,6 @@ HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type)
return false;
}
/*
* Check if a value can be indexed
*/
void
HnswCheckValue(Datum value, HnswType type)
{
if (type == HNSW_TYPE_SPARSEVEC)
{
SparseVector *vec = DatumGetSparseVector(value);
if (vec->nnz > HNSW_MAX_NNZ)
elog(ERROR, "sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ);
}
}
/*
* New buffer
*/
@@ -663,12 +575,7 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
/* Calculate distance */
if (distance != NULL)
{
if (DatumGetPointer(*q) == NULL)
*distance = 0;
else
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
}
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
UnlockReleaseBuffer(buf);
}
@@ -953,15 +860,12 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
static int
#if PG_VERSION_NUM >= 130000
CompareCandidateDistances(const ListCell *a, const ListCell *b)
{
HnswCandidate *hca = lfirst(a);
HnswCandidate *hcb = lfirst(b);
#else
CompareCandidateDistances(const void *a, const void *b)
{
HnswCandidate *hca = lfirst(*(ListCell **) a);
HnswCandidate *hcb = lfirst(*(ListCell **) b);
#endif
{
HnswCandidate *hca = lfirst((ListCell *) a);
HnswCandidate *hcb = lfirst((ListCell *) b);
if (hca->distance < hcb->distance)
return 1;
@@ -984,15 +888,12 @@ CompareCandidateDistances(const void *a, const void *b)
static int
#if PG_VERSION_NUM >= 130000
CompareCandidateDistancesOffset(const ListCell *a, const ListCell *b)
{
HnswCandidate *hca = lfirst(a);
HnswCandidate *hcb = lfirst(b);
#else
CompareCandidateDistancesOffset(const void *a, const void *b)
{
HnswCandidate *hca = lfirst(*(ListCell **) a);
HnswCandidate *hcb = lfirst(*(ListCell **) b);
#endif
{
HnswCandidate *hca = lfirst((ListCell *) a);
HnswCandidate *hcb = lfirst((ListCell *) b);
if (hca->distance < hcb->distance)
return 1;
@@ -1051,9 +952,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
{
List *r = NIL;
List *w = list_copy(c);
HnswCandidate **wd;
int wdlen = 0;
int wdoff = 0;
pairingheap *wd;
HnswNeighborArray *neighbors = HnswGetNeighbors(base, e2, lc);
bool mustCalculate = !neighbors->closerSet;
List *added = NIL;
@@ -1062,7 +961,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
if (list_length(w) <= lm)
return w;
wd = palloc(sizeof(HnswCandidate *) * list_length(w));
wd = pairingheap_allocate(CompareNearestCandidates, NULL);
/* Ensure order of candidates is deterministic for closer caching */
if (sortCandidates)
@@ -1128,21 +1027,21 @@ SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid col
if (e->closer)
r = lappend(r, e);
else
wd[wdlen++] = e;
pairingheap_add(wd, &(CreatePairingHeapNode(e)->ph_node));
}
/* Cached value can only be used in future if sorted deterministically */
neighbors->closerSet = sortCandidates;
/* Keep pruned connections */
while (wdoff < wdlen && list_length(r) < lm)
r = lappend(r, wd[wdoff++]);
while (!pairingheap_is_empty(wd) && list_length(r) < lm)
r = lappend(r, ((HnswPairingHeapNode *) pairingheap_remove_first(wd))->inner);
/* Return pruned for update connections */
if (pruned != NULL)
{
if (wdoff < wdlen)
*pruned = wd[wdoff];
if (!pairingheap_is_empty(wd))
*pruned = ((HnswPairingHeapNode *) pairingheap_first(wd))->inner;
else
*pruned = linitial(w);
}

View File

@@ -57,7 +57,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
*/
if (buildstate->kmeansnormprocinfo != NULL)
{
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value, buildstate->type))
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value, buildstate->normvec))
return;
}
@@ -105,7 +105,7 @@ SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Add sample */
AddSample(values, buildstate);
AddSample(values, state);
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
@@ -153,7 +153,7 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
/* Normalize if needed */
if (buildstate->normprocinfo != NULL)
{
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->type))
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->normvec))
return;
}
@@ -312,39 +312,25 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
}
}
/*
* Get max dimensions
*/
static int
GetMaxDimensions(IvfflatType type)
{
return IVFFLAT_MAX_DIM;
}
/*
* Initialize the build state
*/
static void
InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo)
{
int maxDimensions;
buildstate->heap = heap;
buildstate->index = index;
buildstate->indexInfo = indexInfo;
buildstate->type = IvfflatGetType(index);
buildstate->lists = IvfflatGetLists(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
maxDimensions = GetMaxDimensions(buildstate->type);
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > maxDimensions)
elog(ERROR, "column cannot have more than %d dimensions for ivfflat index", maxDimensions);
if (buildstate->dimensions > IVFFLAT_MAX_DIM)
elog(ERROR, "column cannot have more than %d dimensions for ivfflat index", IVFFLAT_MAX_DIM);
buildstate->reltuples = 0;
buildstate->indtuples = 0;
@@ -370,6 +356,9 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions);
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
/* Reuse for each tuple */
buildstate->normvec = InitVector(buildstate->dimensions);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat build temporary context",
ALLOCSET_DEFAULT_SIZES);
@@ -391,6 +380,7 @@ FreeBuildState(IvfflatBuildState * buildstate)
{
VectorArrayFree(buildstate->centers);
pfree(buildstate->listInfo);
pfree(buildstate->normvec);
#ifdef IVFFLAT_KMEANS_DEBUG
pfree(buildstate->listSums);

View File

@@ -43,11 +43,6 @@
#define IVFFLAT_MAX_LISTS 32768
#define IVFFLAT_DEFAULT_PROBES 1
typedef enum IvfflatType
{
IVFFLAT_TYPE_VECTOR
} IvfflatType;
/* Build phases */
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
#define PROGRESS_IVFFLAT_PHASE_KMEANS 2
@@ -158,7 +153,6 @@ typedef struct IvfflatBuildState
Relation heap;
Relation index;
IndexInfo *indexInfo;
IvfflatType type;
/* Settings */
int dimensions;
@@ -178,6 +172,7 @@ typedef struct IvfflatBuildState
VectorArray samples;
VectorArray centers;
ListInfo *listInfo;
Vector *normvec;
#ifdef IVFFLAT_KMEANS_DEBUG
double inertia;
@@ -272,8 +267,7 @@ void VectorArrayFree(VectorArray arr);
void PrintVectorArray(char *msg, VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
IvfflatType IvfflatGetType(Relation index);
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, IvfflatType type);
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);

View File

@@ -85,7 +85,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
if (normprocinfo != NULL)
{
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value, IvfflatGetType(index)))
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value, NULL))
return;
}

View File

@@ -268,7 +268,6 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
if (so->first)
{
Datum value;
IvfflatType type = IvfflatGetType(scan->indexRelation);
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
@@ -283,12 +282,7 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
elog(ERROR, "non-MVCC snapshots are not supported with ivfflat");
if (scan->orderByData->sk_flags & SK_ISNULL)
{
if (type == IVFFLAT_TYPE_VECTOR)
value = PointerGetDatum(InitVector(so->dimensions));
else
elog(ERROR, "Unsupported type");
}
value = PointerGetDatum(InitVector(so->dimensions));
else
{
value = scan->orderByData->sk_argument;
@@ -299,7 +293,7 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
/* Fine if normalization fails */
if (so->normprocinfo != NULL)
IvfflatNormValue(so->normprocinfo, so->collation, &value, type);
IvfflatNormValue(so->normprocinfo, so->collation, &value, NULL);
}
IvfflatBench("GetScanLists", GetScanLists(scan, value));

View File

@@ -66,15 +66,6 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
return index_getprocinfo(index, 1, procnum);
}
/*
* Get type
*/
IvfflatType
IvfflatGetType(Relation index)
{
return IVFFLAT_TYPE_VECTOR;
}
/*
* Divide by the norm
*
@@ -84,24 +75,21 @@ IvfflatGetType(Relation index)
* if it's different than the original value
*/
bool
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, IvfflatType type)
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result)
{
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
if (norm > 0)
{
if (type == IVFFLAT_TYPE_VECTOR)
{
Vector *v = DatumGetVector(*value);
Vector *result = InitVector(v->dim);
Vector *v = DatumGetVector(*value);
for (int i = 0; i < v->dim; i++)
result->x[i] = v->x[i] / norm;
if (result == NULL)
result = InitVector(v->dim);
*value = PointerGetDatum(result);
}
else
elog(ERROR, "Unsupported type");
for (int i = 0; i < v->dim; i++)
result->x[i] = v->x[i] / norm;
*value = PointerGetDatum(result);
return true;
}

View File

@@ -1,788 +0,0 @@
#include "postgres.h"
#include <limits.h>
#include <math.h>
#include "fmgr.h"
#include "libpq/pqformat.h"
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "vector.h"
#if PG_VERSION_NUM >= 120000
#include "common/shortest_dec.h"
#include "utils/float.h"
#else
#include <float.h>
#include "utils/builtins.h"
#endif
/*
* Ensure same dimensions
*/
static inline void
CheckDims(SparseVector * a, SparseVector * b)
{
if (a->dim != b->dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("different sparsevec dimensions %d and %d", a->dim, b->dim)));
}
/*
* Ensure expected dimensions
*/
static inline void
CheckExpectedDim(int32 typmod, int dim)
{
if (typmod != -1 && typmod != dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("expected %d dimensions, not %d", typmod, dim)));
}
/*
* Ensure valid dimensions
*/
static inline void
CheckDim(int dim)
{
if (dim < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec must have at least 1 dimension")));
if (dim > SPARSEVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more than %d dimensions", SPARSEVEC_MAX_DIM)));
}
/*
* Ensure valid nnz
*/
static inline void
CheckNnz(int nnz, int dim)
{
if (nnz < 0)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec cannot have negative number of elements")));
if (nnz > SPARSEVEC_MAX_NNZ)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more than %d non-zero elements", SPARSEVEC_MAX_NNZ)));
if (nnz > dim)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more elements than dimensions")));
}
/*
* Ensure valid index
*/
static inline void
CheckIndex(int32 *indices, int i, int dim)
{
int32 index = indices[i];
if (index < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("index must be greater than zero")));
if (index > dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("index must be less than or equal to dimensions")));
if (i > 0)
{
if (index < indices[i - 1])
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("indexes must be in ascending order")));
if (index == indices[i - 1])
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("indexes must not contain duplicates")));
}
}
/*
* Ensure finite element
*/
static inline void
CheckElement(float value)
{
if (isnan(value))
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("NaN not allowed in sparsevec")));
if (isinf(value))
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("infinite value not allowed in sparsevec")));
}
/*
* Allocate and initialize a new sparse vector
*/
SparseVector *
InitSparseVector(int dim, int nnz)
{
SparseVector *result;
int size;
size = SPARSEVEC_SIZE(nnz);
result = (SparseVector *) palloc0(size);
SET_VARSIZE(result, size);
result->dim = dim;
result->nnz = nnz;
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
sparsevec_isspace(char ch)
{
if (ch == ' ' ||
ch == '\t' ||
ch == '\n' ||
ch == '\r' ||
ch == '\v' ||
ch == '\f')
return true;
return false;
}
/*
* Convert textual representation to internal representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_in);
Datum
sparsevec_in(PG_FUNCTION_ARGS)
{
char *lit = PG_GETARG_CSTRING(0);
int32 typmod = PG_GETARG_INT32(2);
int dim;
char *pt;
char *stringEnd;
SparseVector *result;
float *rvalues;
char *litcopy = pstrdup(lit);
char *str = litcopy;
int32 *indices;
float *values;
int maxNnz;
int nnz = 0;
maxNnz = 1;
pt = str;
while (*pt != '\0')
{
if (*pt == ',')
maxNnz++;
pt++;
}
if (maxNnz > SPARSEVEC_MAX_NNZ)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more than %d non-zero elements", SPARSEVEC_MAX_NNZ)));
indices = palloc(maxNnz * sizeof(int32));
values = palloc(maxNnz * sizeof(float));
while (sparsevec_isspace(*str))
str++;
if (*str != '{')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed sparsevec literal: \"%s\"", lit),
errdetail("Vector contents must start with \"{\".")));
str++;
pt = strtok(str, ",");
stringEnd = pt;
while (pt != NULL && *stringEnd != '}')
{
long index;
float value;
/* TODO Better error */
if (nnz == maxNnz)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("ran out of buffer: \"%s\"", lit)));
while (sparsevec_isspace(*pt))
pt++;
/* Check for empty string like float4in */
if (*pt == '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Use similar logic as int2vectorin */
errno = 0;
index = strtol(pt, &stringEnd, 10);
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
if (errno == ERANGE || index < 1 || index > INT_MAX)
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("index \"%ld\" is out of range for type sparsevec", index)));
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
while (sparsevec_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != ':')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
stringEnd++;
while (sparsevec_isspace(*stringEnd))
stringEnd++;
errno = 0;
pt = stringEnd;
value = strtof(pt, &stringEnd);
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Check for range error like float4in */
if (errno == ERANGE && (value == 0 || isinf(value)))
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("\"%s\" is out of range for type sparsevec", pt)));
/* TODO Decide whether to store zero values */
if (value != 0)
{
indices[nnz] = index;
values[nnz] = value;
nnz++;
}
if (*stringEnd != '\0' && *stringEnd != '}')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
pt = strtok(NULL, ",");
}
if (stringEnd == NULL || *stringEnd != '}')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed sparsevec literal: \"%s\"", lit),
errdetail("Unexpected end of input.")));
stringEnd++;
if (*stringEnd != '/')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed sparsevec literal: \"%s\"", lit),
errdetail("Unexpected end of input.")));
stringEnd++;
/* Use similar logic as int2vectorin */
errno = 0;
pt = stringEnd;
dim = strtol(pt, &stringEnd, 10);
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Only whitespace is allowed after the closing brace */
while (sparsevec_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed sparsevec literal: \"%s\"", lit),
errdetail("Junk after closing.")));
pfree(litcopy);
CheckDim(dim);
CheckExpectedDim(typmod, dim);
result = InitSparseVector(dim, nnz);
rvalues = SPARSEVEC_VALUES(result);
for (int i = 0; i < nnz; i++)
{
result->indices[i] = indices[i];
rvalues[i] = values[i];
CheckIndex(result->indices, i, dim);
CheckElement(rvalues[i]);
}
PG_RETURN_POINTER(result);
}
#define AppendChar(ptr, c) (*(ptr)++ = (c))
#define AppendFloat(ptr, f) ((ptr) += float_to_shortest_decimal_bufn((f), (ptr)))
#if PG_VERSION_NUM >= 140000
#define AppendInt(ptr, i) ((ptr) += pg_ltoa((i), (ptr)))
#else
#define AppendInt(ptr, i) \
do { \
pg_ltoa(i, ptr); \
while (*ptr != '\0') \
ptr++; \
} while (0)
#endif
/*
* Convert internal representation to textual representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_out);
Datum
sparsevec_out(PG_FUNCTION_ARGS)
{
SparseVector *sparsevec = PG_GETARG_SPARSEVEC_P(0);
float *values = SPARSEVEC_VALUES(sparsevec);
char *buf;
char *ptr;
/*
* Need:
*
* nnz * 10 bytes for index (positive integer)
*
* nnz bytes for :
*
* nnz * (FLOAT_SHORTEST_DECIMAL_LEN - 1) bytes for
* float_to_shortest_decimal_bufn
*
* nnz - 1 bytes for ,
*
* 10 bytes for dimensions
*
* 4 bytes for {, }, /, and \0
*/
buf = (char *) palloc((11 + FLOAT_SHORTEST_DECIMAL_LEN) * sparsevec->nnz + 13);
ptr = buf;
AppendChar(ptr, '{');
for (int i = 0; i < sparsevec->nnz; i++)
{
if (i > 0)
AppendChar(ptr, ',');
AppendInt(ptr, sparsevec->indices[i]);
AppendChar(ptr, ':');
AppendFloat(ptr, values[i]);
}
AppendChar(ptr, '}');
AppendChar(ptr, '/');
AppendInt(ptr, sparsevec->dim);
*ptr = '\0';
PG_FREE_IF_COPY(sparsevec, 0);
PG_RETURN_CSTRING(buf);
}
/*
* Convert type modifier
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
Datum
sparsevec_typmod_in(PG_FUNCTION_ARGS)
{
ArrayType *ta = PG_GETARG_ARRAYTYPE_P(0);
int32 *tl;
int n;
tl = ArrayGetIntegerTypmods(ta, &n);
if (n != 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("invalid type modifier")));
if (*tl < 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions for type sparsevec must be at least 1")));
if (*tl > SPARSEVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions for type sparsevec cannot exceed %d", SPARSEVEC_MAX_DIM)));
PG_RETURN_INT32(*tl);
}
/*
* Convert external binary representation to internal representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_recv);
Datum
sparsevec_recv(PG_FUNCTION_ARGS)
{
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
int32 typmod = PG_GETARG_INT32(2);
SparseVector *result;
int32 dim;
int32 nnz;
int32 unused;
float *values;
dim = pq_getmsgint(buf, sizeof(int32));
nnz = pq_getmsgint(buf, sizeof(int32));
unused = pq_getmsgint(buf, sizeof(int32));
CheckDim(dim);
CheckNnz(nnz, dim);
CheckExpectedDim(typmod, dim);
if (unused != 0)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("expected unused to be 0, not %d", unused)));
result = InitSparseVector(dim, nnz);
values = SPARSEVEC_VALUES(result);
for (int i = 0; i < nnz; i++)
{
result->indices[i] = pq_getmsgint(buf, sizeof(int32));
CheckIndex(result->indices, i, dim);
}
for (int i = 0; i < nnz; i++)
{
values[i] = pq_getmsgfloat4(buf);
CheckElement(values[i]);
}
PG_RETURN_POINTER(result);
}
/*
* Convert internal representation to the external binary representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_send);
Datum
sparsevec_send(PG_FUNCTION_ARGS)
{
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
float *values = SPARSEVEC_VALUES(svec);
StringInfoData buf;
pq_begintypsend(&buf);
pq_sendint(&buf, svec->dim, sizeof(int32));
pq_sendint(&buf, svec->nnz, sizeof(int32));
pq_sendint(&buf, svec->unused, sizeof(int32));
for (int i = 0; i < svec->nnz; i++)
pq_sendint(&buf, svec->indices[i], sizeof(int32));
for (int i = 0; i < svec->nnz; i++)
pq_sendfloat4(&buf, values[i]);
PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
}
/*
* Convert sparse vector to sparse vector
* This is needed to check the type modifier
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec);
Datum
sparsevec(PG_FUNCTION_ARGS)
{
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
CheckExpectedDim(typmod, svec->dim);
PG_RETURN_POINTER(svec);
}
/*
* Convert dense vector to sparse vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_sparsevec);
Datum
vector_to_sparsevec(PG_FUNCTION_ARGS)
{
Vector *vec = PG_GETARG_VECTOR_P(0);
int32 typmod = PG_GETARG_INT32(1);
SparseVector *result;
int dim = vec->dim;
int nnz = 0;
float *values;
int j = 0;
CheckDim(dim);
CheckExpectedDim(typmod, dim);
for (int i = 0; i < dim; i++)
{
if (vec->x[i] != 0)
nnz++;
}
result = InitSparseVector(dim, nnz);
values = SPARSEVEC_VALUES(result);
for (int i = 0; i < dim; i++)
{
if (vec->x[i] != 0)
{
/* Safety check */
if (j == nnz)
elog(ERROR, "safety check failed");
result->indices[j] = i + 1;
values[j] = vec->x[i];
j++;
}
}
PG_RETURN_POINTER(result);
}
/*
* Get the L2 squared distance between sparse vectors
*/
static double
l2_distance_squared_internal(SparseVector * a, SparseVector * b)
{
float *ax = SPARSEVEC_VALUES(a);
float *bx = SPARSEVEC_VALUES(b);
double distance = 0.0;
int bpos = 0;
for (int i = 0; i < a->nnz; i++)
{
int ai = a->indices[i];
int bi = -1;
for (int j = bpos; j < b->nnz; j++)
{
bi = b->indices[j];
if (ai == bi)
{
double diff = ax[i] - bx[j];
distance += diff * diff;
}
else if (ai > bi)
distance += bx[j] * bx[j];
/* Update start for next iteration */
if (ai >= bi)
bpos = j + 1;
/* Found or passed it */
if (bi >= ai)
break;
}
if (ai != bi)
distance += ax[i] * ax[i];
}
for (int j = bpos; j < b->nnz; j++)
distance += bx[j] * bx[j];
return distance;
}
/*
* Get the L2 distance between sparse vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_distance);
Datum
sparsevec_l2_distance(PG_FUNCTION_ARGS)
{
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(sqrt(l2_distance_squared_internal(a, b)));
}
/*
* Get the L2 squared distance between sparse vectors
* This saves a sqrt calculation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance);
Datum
sparsevec_l2_squared_distance(PG_FUNCTION_ARGS)
{
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(l2_distance_squared_internal(a, b));
}
/*
* Get the inner product of two sparse vectors
*/
static double
inner_product_internal(SparseVector * a, SparseVector * b)
{
float *ax = SPARSEVEC_VALUES(a);
float *bx = SPARSEVEC_VALUES(b);
double distance = 0.0;
int bpos = 0;
for (int i = 0; i < a->nnz; i++)
{
int ai = a->indices[i];
for (int j = bpos; j < b->nnz; j++)
{
int bi = b->indices[j];
/* Only update when the same index */
if (ai == bi)
distance += ax[i] * bx[j];
/* Update start for next iteration */
if (ai >= bi)
bpos = j + 1;
/* Found or passed it */
if (bi >= ai)
break;
}
}
return distance;
}
/*
* Get the inner product of two sparse vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_inner_product);
Datum
sparsevec_inner_product(PG_FUNCTION_ARGS)
{
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(inner_product_internal(a, b));
}
/*
* Get the negative inner product of two sparse vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product);
Datum
sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
{
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(-inner_product_internal(a, b));
}
/*
* Get the cosine distance between two sparse vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cosine_distance);
Datum
sparsevec_cosine_distance(PG_FUNCTION_ARGS)
{
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
float *ax = SPARSEVEC_VALUES(a);
float *bx = SPARSEVEC_VALUES(b);
float norma = 0.0;
float normb = 0.0;
double similarity;
CheckDims(a, b);
similarity = inner_product_internal(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->nnz; i++)
norma += ax[i] * ax[i];
/* Auto-vectorized */
for (int i = 0; i < b->nnz; i++)
normb += bx[i] * bx[i];
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
similarity /= sqrt((double) norma * (double) normb);
#ifdef _MSC_VER
/* /fp:fast may not propagate NaN */
if (isnan(similarity))
PG_RETURN_FLOAT8(NAN);
#endif
/* Keep in range */
if (similarity > 1)
similarity = 1.0;
else if (similarity < -1)
similarity = -1.0;
PG_RETURN_FLOAT8(1.0 - similarity);
}
/*
* Get the L2 norm of a sparse vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_norm);
Datum
sparsevec_norm(PG_FUNCTION_ARGS)
{
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
float *ax = SPARSEVEC_VALUES(a);
double norm = 0.0;
/* Auto-vectorized */
for (int i = 0; i < a->nnz; i++)
norm += (double) ax[i] * (double) ax[i];
PG_RETURN_FLOAT8(sqrt(norm));
}

View File

@@ -1,25 +0,0 @@
#ifndef SPARSEVEC_H
#define SPARSEVEC_H
#define SPARSEVEC_MAX_DIM 100000
#define SPARSEVEC_MAX_NNZ 16000
/* Ensure values are aligned */
#define SPARSEVEC_SIZE(_nnz) (offsetof(SparseVector, indices) + MAXALIGN((_nnz) * sizeof(int32)) + (_nnz * sizeof(float)))
#define SPARSEVEC_VALUES(x) ((float *) (((char *) (x)) + offsetof(SparseVector, indices) + MAXALIGN((x)->nnz * sizeof(int32))))
#define DatumGetSparseVector(x) ((SparseVector *) PG_DETOAST_DATUM(x))
#define PG_GETARG_SPARSEVEC_P(x) DatumGetSparseVector(PG_GETARG_DATUM(x))
#define PG_RETURN_SPARSEVEC_P(x) PG_RETURN_POINTER(x)
typedef struct SparseVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int32 dim; /* number of dimensions */
int32 nnz;
int32 unused;
int32 indices[FLEXIBLE_ARRAY_MEMBER];
} SparseVector;
SparseVector *InitSparseVector(int dim, int nnz);
#endif

View File

@@ -2,17 +2,14 @@
#include <math.h>
#include "bitvector.h"
#include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "fmgr.h"
#include "halfvec.h"
#include "hnsw.h"
#include "ivfflat.h"
#include "lib/stringinfo.h"
#include "libpq/pqformat.h"
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
@@ -200,8 +197,6 @@ vector_in(PG_FUNCTION_ARGS)
while (pt != NULL && *stringEnd != ']')
{
float val;
if (dim == VECTOR_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
@@ -217,23 +212,15 @@ vector_in(PG_FUNCTION_ARGS)
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
/* Use strtof like float4in to avoid a double-rounding problem */
errno = 0;
val = strtof(pt, &stringEnd);
x[dim] = strtof(pt, &stringEnd);
CheckElement(x[dim]);
dim++;
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
/* Check for range error like float4in */
if (errno == ERANGE && isinf(val))
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("\"%s\" is out of range for type vector", pt)));
CheckElement(val);
x[dim++] = val;
while (vector_isspace(*stringEnd))
stringEnd++;
@@ -288,9 +275,6 @@ vector_in(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
#define AppendChar(ptr, c) (*(ptr)++ = (c))
#define AppendFloat(ptr, f) ((ptr) += float_to_shortest_decimal_bufn((f), (ptr)))
/*
* Convert internal representation to textual representation
*/
@@ -302,6 +286,7 @@ vector_out(PG_FUNCTION_ARGS)
int dim = vector->dim;
char *buf;
char *ptr;
int n;
/*
* Need:
@@ -316,17 +301,21 @@ vector_out(PG_FUNCTION_ARGS)
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
ptr = buf;
AppendChar(ptr, '[');
*ptr = '[';
ptr++;
for (int i = 0; i < dim; i++)
{
if (i > 0)
AppendChar(ptr, ',');
{
*ptr = ',';
ptr++;
}
AppendFloat(ptr, vector->x[i]);
n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
ptr += n;
}
AppendChar(ptr, ']');
*ptr = ']';
ptr++;
*ptr = '\0';
PG_FREE_IF_COPY(vector, 0);
@@ -543,28 +532,6 @@ vector_to_float4(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Convert half vector to vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_vector);
Datum
halfvec_to_vector(PG_FUNCTION_ARGS)
{
HalfVector *vec = PG_GETARG_HALFVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
Vector *result;
CheckDim(vec->dim);
CheckExpectedDim(typmod, vec->dim);
result = InitVector(vec->dim);
for (int i = 0; i < vec->dim; i++)
result->x[i] = HalfToFloat4(vec->x[i]);
PG_RETURN_POINTER(result);
}
/*
* Get the L2 distance between vectors
*/
@@ -893,56 +860,6 @@ vector_mul(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Quantize a vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(quantize_binary);
Datum
quantize_binary(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
for (int i = 0; i < a->dim; i++)
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);
}
/*
* Get a subvector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
Datum
subvector(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
int32 start = PG_GETARG_INT32(1);
int32 count = PG_GETARG_INT32(2);
int32 end = start + count;
float *ax = a->x;
Vector *result;
int dim;
/* Indexing starts at 1, like substring */
if (start < 1)
start = 1;
if (end > a->dim)
end = a->dim + 1;
dim = end - start;
CheckDim(dim);
result = InitVector(dim);
for (int i = 0; i < dim; i++)
result->x[i] = ax[start - 1 + i];
PG_RETURN_POINTER(result);
}
/*
* Internal helper to compare vectors
*/
@@ -980,6 +897,9 @@ vector_lt(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
}
@@ -993,6 +913,9 @@ vector_le(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
}
@@ -1006,6 +929,9 @@ vector_eq(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
}
@@ -1019,6 +945,9 @@ vector_ne(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
}
@@ -1032,6 +961,9 @@ vector_ge(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
}
@@ -1045,6 +977,9 @@ vector_gt(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
}
@@ -1225,26 +1160,3 @@ vector_avg(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Convert sparse vector to dense vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_vector);
Datum
sparsevec_to_vector(PG_FUNCTION_ARGS)
{
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
Vector *result;
int dim = svec->dim;
float *values = SPARSEVEC_VALUES(svec);
CheckDim(dim);
CheckExpectedDim(typmod, dim);
result = InitVector(dim);
for (int i = 0; i < svec->nnz; i++)
result->x[svec->indices[i] - 1] = values[i];
PG_RETURN_POINTER(result);
}

View File

@@ -1,104 +0,0 @@
SELECT hamming_distance('111', '111');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance('111', '110');
hamming_distance
------------------
1
(1 row)
SELECT hamming_distance('111', '100');
hamming_distance
------------------
2
(1 row)
SELECT hamming_distance('111', '000');
hamming_distance
------------------
3
(1 row)
SELECT hamming_distance('10101010101010101010', '01010101010101010101');
hamming_distance
------------------
20
(1 row)
SELECT hamming_distance('', '');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance('111', '00');
ERROR: different bit lengths 3 and 2
SELECT hamming_distance('111', '000'::varbit(4));
hamming_distance
------------------
3
(1 row)
SELECT hamming_distance('111', '0000'::varbit(4));
ERROR: different bit lengths 3 and 4
SELECT jaccard_distance('1111', '1111');
jaccard_distance
------------------
0
(1 row)
SELECT jaccard_distance('1111', '1110');
jaccard_distance
------------------
0.25
(1 row)
SELECT jaccard_distance('1111', '1100');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance('1111', '1000');
jaccard_distance
------------------
0.75
(1 row)
SELECT jaccard_distance('1111', '0000');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1100', '1000');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance('10101010101010101010', '01010101010101010101');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('', '');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1111', '000');
ERROR: different bit lengths 4 and 3
SELECT jaccard_distance('1111', '0000'::varbit(5));
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1111', '00000'::varbit(5));
ERROR: different bit lengths 4 and 5

View File

@@ -46,30 +46,6 @@ SELECT '[1,2,3]'::vector::real[];
{1,2,3}
(1 row)
SELECT '[1,2,3]'::vector::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[65520]'::vector::halfvec;
ERROR: infinite value not allowed in halfvec
SELECT '[1e-8]'::vector::halfvec;
halfvec
---------
[0]
(1 row)
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
ERROR: vector cannot have more than 16000 dimensions
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -1,15 +1,15 @@
CREATE TABLE t (val vector(3), val2 halfvec(3));
INSERT INTO t (val, val2) VALUES ('[0,0,0]', '[0,0,0]'), ('[1,2,3]', '[1,2,3]'), ('[1,1,1]', '[1,1,1]'), (NULL, NULL);
CREATE TABLE t2 (val vector(3), val2 halfvec(3));
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3));
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val | val2
---------+---------
[0,0,0] | [0,0,0]
[1,1,1] | [1,1,1]
[1,2,3] | [1,2,3]
|
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;

View File

@@ -24,30 +24,6 @@ SELECT '[1e37]'::vector * '[1e37]';
ERROR: value out of range: overflow
SELECT '[1e-37]'::vector * '[1e-37]';
ERROR: value out of range: underflow
SELECT '[1,2,3]'::vector < '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector < '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector = '[1,2,3]';
?column?
----------
@@ -55,47 +31,7 @@ SELECT '[1,2,3]'::vector = '[1,2,3]';
(1 row)
SELECT '[1,2,3]'::vector = '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector > '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector > '[1,2]';
?column?
----------
t
(1 row)
ERROR: different vector dimensions 3 and 2
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
vector_cmp
------------
@@ -168,152 +104,110 @@ SELECT vector_norm('[3e37,4e37]')::real;
5e+37
(1 row)
SELECT l2_distance('[0,0]'::vector, '[3,4]');
SELECT l2_distance('[0,0]', '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::vector, '[0,1]');
SELECT l2_distance('[0,0]', '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::vector, '[3]');
SELECT l2_distance('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
SELECT l2_distance('[3e38]', '[-3e38]');
l2_distance
-------------
Infinity
(1 row)
SELECT inner_product('[1,2]'::vector, '[3,4]');
SELECT inner_product('[1,2]', '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::vector, '[3]');
SELECT inner_product('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1
SELECT inner_product('[3e38]'::vector, '[3e38]');
SELECT inner_product('[3e38]', '[3e38]');
inner_product
---------------
Infinity
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
SELECT cosine_distance('[1,2]', '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
SELECT cosine_distance('[1,2]', '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
SELECT cosine_distance('[1,1]', '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
SELECT cosine_distance('[1,0]', '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
SELECT cosine_distance('[1,1]', '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[3]');
SELECT cosine_distance('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
SELECT cosine_distance('[3e38]', '[3e38]');
cosine_distance
-----------------
NaN
(1 row)
SELECT l1_distance('[0,0]'::vector, '[3,4]');
SELECT l1_distance('[0,0]', '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::vector, '[0,1]');
SELECT l1_distance('[0,0]', '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::vector, '[3]');
SELECT l1_distance('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
SELECT l1_distance('[3e38]', '[-3e38]');
l1_distance
-------------
Infinity
(1 row)
SELECT quantize_binary('[1,0,-1]'::vector);
quantize_binary
-----------------
100
(1 row)
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
quantize_binary
-----------------
01001110101
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
subvector
-----------
[1,2,3]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
subvector
-----------
[3,4]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, -1, 3);
subvector
-----------
[1]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 3, 9);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 0);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
ERROR: vector must have at least 1 dimension
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
avg
-----------

View File

@@ -1,146 +0,0 @@
SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::halfvec, '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT '[0,0]'::halfvec <-> '[3,4]';
?column?
----------
5
(1 row)
SELECT inner_product('[1,2]'::halfvec, '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT inner_product('[65504]'::halfvec, '[65504]');
inner_product
---------------
4290774016
(1 row)
SELECT '[1,2]'::halfvec <#> '[3,4]';
?column?
----------
-11
(1 row)
SELECT cosine_distance('[1,2]'::halfvec, '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::halfvec, '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::halfvec, '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::halfvec, '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::halfvec, '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT cosine_distance('[1,1]'::halfvec, '[1.1,1.1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,1]'::halfvec, '[-1.1,-1.1]');
cosine_distance
-----------------
2
(1 row)
SELECT '[1,2]'::halfvec <=> '[2,4]';
?column?
----------
0
(1 row)
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT quantize_binary('[1,0,-1]'::halfvec);
quantize_binary
-----------------
100
(1 row)
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
quantize_binary
-----------------
01001110101
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
subvector
-----------
[1,2,3]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
subvector
-----------
[3,4]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 3);
subvector
-----------
[1]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
ERROR: halfvec must have at least 1 dimension

View File

@@ -1,151 +0,0 @@
SELECT '[1,2,3]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::halfvec;
halfvec
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::halfvec;
halfvec
------------
[1.234375]
(1 row)
SELECT '[hello,1]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[hello,1]"
LINE 1: SELECT '[hello,1]'::halfvec;
^
SELECT '[NaN,1]'::halfvec;
ERROR: NaN not allowed in halfvec
LINE 1: SELECT '[NaN,1]'::halfvec;
^
SELECT '[Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[Infinity,1]'::halfvec;
^
SELECT '[-Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[-Infinity,1]'::halfvec;
^
SELECT '[65519,-65519]'::halfvec;
halfvec
----------------
[65504,-65504]
(1 row)
SELECT '[65520,-65520]'::halfvec;
ERROR: "65520" is out of range for type halfvec
LINE 1: SELECT '[65520,-65520]'::halfvec;
^
SELECT '[1e-8,-1e-8]'::halfvec;
ERROR: "1e-8" is out of range for type halfvec
LINE 1: SELECT '[1e-8,-1e-8]'::halfvec;
^
SELECT '[4e38,1]'::halfvec;
ERROR: "4e38" is out of range for type halfvec
LINE 1: SELECT '[4e38,1]'::halfvec;
^
SELECT '[1e-46,1]'::halfvec;
ERROR: "1e-46" is out of range for type halfvec
LINE 1: SELECT '[1e-46,1]'::halfvec;
^
SELECT '[1,2,3'::halfvec;
ERROR: malformed halfvec literal: "[1,2,3"
LINE 1: SELECT '[1,2,3'::halfvec;
^
DETAIL: Unexpected end of input.
SELECT '[1,2,3]9'::halfvec;
ERROR: malformed halfvec literal: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::halfvec;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::halfvec;
ERROR: malformed halfvec literal: "1,2,3"
LINE 1: SELECT '1,2,3'::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT ''::halfvec;
ERROR: malformed halfvec literal: ""
LINE 1: SELECT ''::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT '['::halfvec;
ERROR: malformed halfvec literal: "["
LINE 1: SELECT '['::halfvec;
^
DETAIL: Unexpected end of input.
SELECT '[,'::halfvec;
ERROR: malformed halfvec literal: "[,"
LINE 1: SELECT '[,'::halfvec;
^
DETAIL: Unexpected end of input.
SELECT '[]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[]'::halfvec;
^
SELECT '[1,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,]"
LINE 1: SELECT '[1,]'::halfvec;
^
SELECT '[1a]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1a]"
LINE 1: SELECT '[1a]'::halfvec;
^
SELECT '[1,,3]'::halfvec;
ERROR: malformed halfvec literal: "[1,,3]"
LINE 1: SELECT '[1,,3]'::halfvec;
^
SELECT '[1, ,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::halfvec;
^
SELECT '[1,2,3]'::halfvec(3);
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::halfvec(3, 2);
^
SELECT '[1,2,3]'::halfvec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::halfvec('a');
^
SELECT '[1,2,3]'::halfvec(0);
ERROR: dimensions for type halfvec must be at least 1
LINE 1: SELECT '[1,2,3]'::halfvec(0);
^
SELECT '[1,2,3]'::halfvec(16001);
ERROR: dimensions for type halfvec cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::halfvec(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::halfvec(2)[];
ERROR: expected 2 dimensions, not 3

View File

@@ -1,27 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
val
-----
111
110
100
000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: type not supported for hnsw index
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
DROP TABLE t;

View File

@@ -1,21 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
val
------
1111
1110
1100
0000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -1,26 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -1,21 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -1,33 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -12,11 +12,14 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
[1,2,4]
(4 rows)
SELECT COUNT(*) FROM t;
count

View File

@@ -1,26 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -1,21 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:4}/3
{1:1,2:2,3:3}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -1,43 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----
(0 rows)
DROP TABLE t;
-- TODO move
CREATE TABLE t (val sparsevec(1001));
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
TRUNCATE t;
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
DROP TABLE t;

View File

@@ -63,25 +63,9 @@ SELECT '[1.5e-38,-1.5e-38]'::vector;
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: "4e38" is out of range for type vector
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[-4e38,1]'::vector;
ERROR: "-4e38" is out of range for type vector
LINE 1: SELECT '[-4e38,1]'::vector;
^
SELECT '[1e-46,1]'::vector;
vector
--------
[0,1]
(1 row)
SELECT '[-1e-46,1]'::vector;
vector
--------
[-0,1]
(1 row)
SELECT '[1,2,3'::vector;
ERROR: malformed vector literal: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector;
@@ -132,30 +116,8 @@ SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
^
SELECT '[1,2,3]'::vector('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::vector('a');
^
SELECT '[1,2,3]'::vector(0);
ERROR: dimensions for type vector must be at least 1
LINE 1: SELECT '[1,2,3]'::vector(0);
^
SELECT '[1,2,3]'::vector(16001);
ERROR: dimensions for type vector cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::vector(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
unnest
---------

View File

@@ -1,62 +0,0 @@
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
l2_distance
-------------
1
(1 row)
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2';
?column?
----------
5
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
inner_product
---------------
10
(1 row)
SELECT sparsevec_negative_inner_product('{1:1,2:2}/2', '{1:2,2:4}/2');
sparsevec_negative_inner_product
----------------------------------
-10
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{1:1}/3');
ERROR: different sparsevec dimensions 2 and 3

View File

@@ -1,62 +0,0 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector;
vector
-----------------
[1.5,0,3.5,0,0]
(1 row)
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(5);
vector
-----------------
[1.5,0,3.5,0,0]
(1 row)
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(4);
ERROR: expected 4 dimensions, not 5
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
sparsevec
-----------------
{2:1.5,4:3.5}/5
(1 row)
SELECT '{1:0,2:1,3:0}/3'::sparsevec;
sparsevec
-----------
{2:1}/3
(1 row)
SELECT '{2:1,1:1}/2'::sparsevec;
ERROR: indexes must be in ascending order
LINE 1: SELECT '{2:1,1:1}/2'::sparsevec;
^
SELECT '{}/5'::sparsevec;
sparsevec
-----------
{}/5
(1 row)
SELECT '{}/-1'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-1'::sparsevec;
^
SELECT '{}/100001'::sparsevec;
ERROR: sparsevec cannot have more than 100000 dimensions
LINE 1: SELECT '{}/100001'::sparsevec;
^
SELECT '{}/16001'::sparsevec::vector;
ERROR: vector cannot have more than 16000 dimensions
SELECT '{0:1}/1'::sparsevec;
ERROR: index "0" is out of range for type sparsevec
LINE 1: SELECT '{0:1}/1'::sparsevec;
^
SELECT '{2:1}/1'::sparsevec;
ERROR: index must be less than or equal to dimensions
LINE 1: SELECT '{2:1}/1'::sparsevec;
^
SELECT '{}/1'::sparsevec(2);
ERROR: expected 2 dimensions, not 1

View File

@@ -1,21 +0,0 @@
SELECT hamming_distance('111', '111');
SELECT hamming_distance('111', '110');
SELECT hamming_distance('111', '100');
SELECT hamming_distance('111', '000');
SELECT hamming_distance('10101010101010101010', '01010101010101010101');
SELECT hamming_distance('', '');
SELECT hamming_distance('111', '00');
SELECT hamming_distance('111', '000'::varbit(4));
SELECT hamming_distance('111', '0000'::varbit(4));
SELECT jaccard_distance('1111', '1111');
SELECT jaccard_distance('1111', '1110');
SELECT jaccard_distance('1111', '1100');
SELECT jaccard_distance('1111', '1000');
SELECT jaccard_distance('1111', '0000');
SELECT jaccard_distance('1100', '1000');
SELECT jaccard_distance('10101010101010101010', '01010101010101010101');
SELECT jaccard_distance('', '');
SELECT jaccard_distance('1111', '000');
SELECT jaccard_distance('1111', '0000'::varbit(5));
SELECT jaccard_distance('1111', '00000'::varbit(5));

View File

@@ -10,12 +10,6 @@ SELECT '{-Infinity}'::real[]::vector;
SELECT '{}'::real[]::vector;
SELECT '{{1}}'::real[]::vector;
SELECT '[1,2,3]'::vector::real[];
SELECT '[1,2,3]'::vector::halfvec;
SELECT '[1,2,3]'::halfvec::vector;
SELECT '[1,2,3]'::vector::halfvec(2);
SELECT '[1,2,3]'::halfvec::vector(2);
SELECT '[65520]'::vector::halfvec;
SELECT '[1e-8]'::vector::halfvec;
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -1,7 +1,7 @@
CREATE TABLE t (val vector(3), val2 halfvec(3));
INSERT INTO t (val, val2) VALUES ('[0,0,0]', '[0,0,0]'), ('[1,2,3]', '[1,2,3]'), ('[1,1,1]', '[1,1,1]'), (NULL, NULL);
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3), val2 halfvec(3));
CREATE TABLE t2 (val vector(3));
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)

62
test/sql/functions.sql Normal file
View File

@@ -0,0 +1,62 @@
SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[3e38]'::vector + '[3e38]';
SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT '[-3e38]'::vector - '[3e38]';
SELECT '[1,2,3]'::vector * '[4,5,6]';
SELECT '[1e37]'::vector * '[1e37]';
SELECT '[1e-37]'::vector * '[1e-37]';
SELECT '[1,2,3]'::vector = '[1,2,3]';
SELECT '[1,2,3]'::vector = '[1,2]';
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
SELECT vector_cmp('[1,2]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[1,2]');
SELECT vector_cmp('[1,2]', '[2,3,4]');
SELECT vector_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT vector_norm('[3,4]');
SELECT vector_norm('[0,1]');
SELECT vector_norm('[3e37,4e37]')::real;
SELECT l2_distance('[0,0]', '[3,4]');
SELECT l2_distance('[0,0]', '[0,1]');
SELECT l2_distance('[1,2]', '[3]');
SELECT l2_distance('[3e38]', '[-3e38]');
SELECT inner_product('[1,2]', '[3,4]');
SELECT inner_product('[1,2]', '[3]');
SELECT inner_product('[3e38]', '[3e38]');
SELECT cosine_distance('[1,2]', '[2,4]');
SELECT cosine_distance('[1,2]', '[0,0]');
SELECT cosine_distance('[1,1]', '[1,1]');
SELECT cosine_distance('[1,0]', '[0,2]');
SELECT cosine_distance('[1,1]', '[-1,-1]');
SELECT cosine_distance('[1,2]', '[3]');
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
SELECT cosine_distance('[3e38]', '[3e38]');
SELECT l1_distance('[0,0]', '[3,4]');
SELECT l1_distance('[0,0]', '[0,1]');
SELECT l1_distance('[1,2]', '[3]');
SELECT l1_distance('[3e38]', '[-3e38]');
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;

View File

@@ -1,34 +0,0 @@
SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
SELECT l2_distance('[0,0]'::halfvec, '[0,1]');
SELECT l2_distance('[1,2]'::halfvec, '[3]');
SELECT '[0,0]'::halfvec <-> '[3,4]';
SELECT inner_product('[1,2]'::halfvec, '[3,4]');
SELECT inner_product('[1,2]'::halfvec, '[3]');
SELECT inner_product('[65504]'::halfvec, '[65504]');
SELECT '[1,2]'::halfvec <#> '[3,4]';
SELECT cosine_distance('[1,2]'::halfvec, '[2,4]');
SELECT cosine_distance('[1,2]'::halfvec, '[0,0]');
SELECT cosine_distance('[1,1]'::halfvec, '[1,1]');
SELECT cosine_distance('[1,0]'::halfvec, '[0,2]');
SELECT cosine_distance('[1,1]'::halfvec, '[-1,-1]');
SELECT cosine_distance('[1,2]'::halfvec, '[3]');
SELECT cosine_distance('[1,1]'::halfvec, '[1.1,1.1]');
SELECT cosine_distance('[1,1]'::halfvec, '[-1.1,-1.1]');
SELECT '[1,2]'::halfvec <=> '[2,4]';
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
SELECT l1_distance('[1,2]'::halfvec, '[3]');
SELECT quantize_binary('[1,0,-1]'::halfvec);
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);

View File

@@ -1,35 +0,0 @@
SELECT '[1,2,3]'::halfvec;
SELECT '[-1,-2,-3]'::halfvec;
SELECT '[1.,2.,3.]'::halfvec;
SELECT ' [ 1, 2 , 3 ] '::halfvec;
SELECT '[1.23456]'::halfvec;
SELECT '[hello,1]'::halfvec;
SELECT '[NaN,1]'::halfvec;
SELECT '[Infinity,1]'::halfvec;
SELECT '[-Infinity,1]'::halfvec;
SELECT '[65519,-65519]'::halfvec;
SELECT '[65520,-65520]'::halfvec;
SELECT '[1e-8,-1e-8]'::halfvec;
SELECT '[4e38,1]'::halfvec;
SELECT '[1e-46,1]'::halfvec;
SELECT '[1,2,3'::halfvec;
SELECT '[1,2,3]9'::halfvec;
SELECT '1,2,3'::halfvec;
SELECT ''::halfvec;
SELECT '['::halfvec;
SELECT '[,'::halfvec;
SELECT '[]'::halfvec;
SELECT '[1,]'::halfvec;
SELECT '[1a]'::halfvec;
SELECT '[1,,3]'::halfvec;
SELECT '[1, ,3]'::halfvec;
SELECT '[1,2,3]'::halfvec(3);
SELECT '[1,2,3]'::halfvec(2);
SELECT '[1,2,3]'::halfvec(3, 2);
SELECT '[1,2,3]'::halfvec('a');
SELECT '[1,2,3]'::halfvec(0);
SELECT '[1,2,3]'::halfvec(16001);
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
SELECT '{"[1,2,3]"}'::halfvec(2)[];

View File

@@ -1,18 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
DROP TABLE t;
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
DROP TABLE t;

View File

@@ -1,12 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
DROP TABLE t;

View File

@@ -1,13 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t;

View File

@@ -1,12 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
DROP TABLE t;

View File

@@ -1,16 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
DROP TABLE t;

View File

@@ -7,7 +7,7 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
SELECT COUNT(*) FROM t;
TRUNCATE t;

View File

@@ -1,13 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
DROP TABLE t;

View File

@@ -1,12 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
DROP TABLE t;

View File

@@ -1,25 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
DROP TABLE t;
-- TODO move
CREATE TABLE t (val sparsevec(1001));
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
TRUNCATE t;
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
DROP TABLE t;

View File

@@ -11,9 +11,6 @@ SELECT '[1.5e38,-1.5e38]'::vector;
SELECT '[1.5e+38,-1.5e+38]'::vector;
SELECT '[1.5e-38,-1.5e-38]'::vector;
SELECT '[4e38,1]'::vector;
SELECT '[-4e38,1]'::vector;
SELECT '[1e-46,1]'::vector;
SELECT '[-1e-46,1]'::vector;
SELECT '[1,2,3'::vector;
SELECT '[1,2,3]9'::vector;
SELECT '1,2,3'::vector;
@@ -25,13 +22,7 @@ SELECT '[1,]'::vector;
SELECT '[1a]'::vector;
SELECT '[1,,3]'::vector;
SELECT '[1, ,3]'::vector;
SELECT '[1,2,3]'::vector(3);
SELECT '[1,2,3]'::vector(2);
SELECT '[1,2,3]'::vector(3, 2);
SELECT '[1,2,3]'::vector('a');
SELECT '[1,2,3]'::vector(0);
SELECT '[1,2,3]'::vector(16001);
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
SELECT '{"[1,2,3]"}'::vector(2)[];

View File

@@ -1,13 +0,0 @@
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2';
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
SELECT sparsevec_negative_inner_product('{1:1,2:2}/2', '{1:2,2:4}/2');
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2');
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2');
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
SELECT cosine_distance('{1:2}/2'::sparsevec, '{1:1}/3');

View File

@@ -1,19 +0,0 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec;
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector;
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(5);
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(4);
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
SELECT '{1:0,2:1,3:0}/3'::sparsevec;
SELECT '{2:1,1:1}/2'::sparsevec;
SELECT '{}/5'::sparsevec;
SELECT '{}/-1'::sparsevec;
SELECT '{}/100001'::sparsevec;
SELECT '{}/16001'::sparsevec::vector;
SELECT '{0:1}/1'::sparsevec;
SELECT '{2:1}/1'::sparsevec;
SELECT '{}/1'::sparsevec(2);

View File

@@ -1,83 +0,0 @@
SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[3e38]'::vector + '[3e38]';
SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT '[-3e38]'::vector - '[3e38]';
SELECT '[1,2,3]'::vector * '[4,5,6]';
SELECT '[1e37]'::vector * '[1e37]';
SELECT '[1e-37]'::vector * '[1e-37]';
SELECT '[1,2,3]'::vector < '[1,2,3]';
SELECT '[1,2,3]'::vector < '[1,2]';
SELECT '[1,2,3]'::vector <= '[1,2,3]';
SELECT '[1,2,3]'::vector <= '[1,2]';
SELECT '[1,2,3]'::vector = '[1,2,3]';
SELECT '[1,2,3]'::vector = '[1,2]';
SELECT '[1,2,3]'::vector != '[1,2,3]';
SELECT '[1,2,3]'::vector != '[1,2]';
SELECT '[1,2,3]'::vector >= '[1,2,3]';
SELECT '[1,2,3]'::vector >= '[1,2]';
SELECT '[1,2,3]'::vector > '[1,2,3]';
SELECT '[1,2,3]'::vector > '[1,2]';
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
SELECT vector_cmp('[1,2]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[1,2]');
SELECT vector_cmp('[1,2]', '[2,3,4]');
SELECT vector_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT vector_norm('[3,4]');
SELECT vector_norm('[0,1]');
SELECT vector_norm('[3e37,4e37]')::real;
SELECT l2_distance('[0,0]'::vector, '[3,4]');
SELECT l2_distance('[0,0]'::vector, '[0,1]');
SELECT l2_distance('[1,2]'::vector, '[3]');
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
SELECT inner_product('[1,2]'::vector, '[3,4]');
SELECT inner_product('[1,2]'::vector, '[3]');
SELECT inner_product('[3e38]'::vector, '[3e38]');
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
SELECT cosine_distance('[1,2]'::vector, '[3]');
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
SELECT l1_distance('[0,0]'::vector, '[3,4]');
SELECT l1_distance('[0,0]'::vector, '[0,1]');
SELECT l1_distance('[1,2]'::vector, '[3]');
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
SELECT quantize_binary('[1,0,-1]'::vector);
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
SELECT subvector('[1,2,3,4,5]'::vector, -1, 3);
SELECT subvector('[1,2,3,4,5]'::vector, 3, 9);
SELECT subvector('[1,2,3,4,5]'::vector, 1, 0);
SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;

View File

@@ -86,7 +86,7 @@ foreach (@queries)
push(@expected, $res);
}
test_recall(0.18, $limit, "before vacuum");
test_recall(0.20, $limit, "before vacuum");
test_recall(0.95, 100, "before vacuum");
# TODO Test concurrent inserts with vacuum

View File

@@ -52,23 +52,43 @@ $node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v sparsevec(3));");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v float4[3]);");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()]::vector::sparsevec FROM generate_series(1, 10000) i;"
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
);
$node->safe_psql("postgres", qq(
CREATE FUNCTION float4_l2_distance(float4[], float4[]) RETURNS float8
AS 'BEGIN RETURN l2_distance(\$1::vector, \$2::vector); END;'
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION float4_l2_squared_distance(float4[], float4[]) RETURNS float8
AS 'BEGIN RETURN vector_l2_squared_distance(\$1::vector, \$2::vector); END;'
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <-> (
LEFTARG = float4[], RIGHTARG = float4[], PROCEDURE = float4_l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR CLASS float4_l2_ops
FOR TYPE float4[] USING hnsw AS
OPERATOR 1 <-> (float4[], float4[]) FOR ORDER BY float_ops,
FUNCTION 1 float4_l2_squared_distance(float4[], float4[]);
));
# Generate queries
for (1 .. 20)
{
my $r1 = rand();
my $r2 = rand();
my $r3 = rand();
push(@queries, "{1:$r1,2:$r2,3:$r3}/3");
push(@queries, "{$r1,$r2,$r3}");
}
# Check each index type
my @operators = ("<->", "<#>", "<=>");
my @opclasses = ("sparsevec_l2_ops", "sparsevec_ip_ops", "sparsevec_cosine_ops");
my @operators = ("<->");
my @opclasses = ("float4_l2_ops");
for my $i (0 .. $#operators)
{
@@ -83,46 +103,11 @@ for my $i (0 .. $#operators)
push(@expected, $res);
}
# Build index serially
$node->safe_psql("postgres", qq(
SET max_parallel_maintenance_workers = 0;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v $opclass);");
# Test approximate results
my $min = $operator eq "<#>" ? 0.80 : 0.99;
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel in memory
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET min_parallel_table_scan_size = 1;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
# Test approximate results
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel on disk
# Set parallel_workers on table to use workers with low maintenance_work_mem
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
ALTER TABLE tst SET (parallel_workers = 2);
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '4MB';
CREATE INDEX idx ON tst USING hnsw (v $opclass);
ALTER TABLE tst RESET (parallel_workers);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
$node->safe_psql("postgres", "DROP INDEX idx;");
}
done_testing();

View File

@@ -1,137 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More;
my $node;
my @queries = ();
my @expected;
my $limit = 20;
my $dim = 52;
my $max = 2**$dim;
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET hnsw.ef_search = 100;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator $queries[0] LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET hnsw.ef_search = 100;
SELECT i FROM tst ORDER BY v $operator $queries[$i] LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my @expected_ids = split("\n", $expected[$i]);
my %expected_set = map { $_ => 1 } @expected_ids;
foreach (@actual_ids)
{
if (exists($expected_set{$_}))
{
$correct++;
}
}
$total += $limit;
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v bit($dim));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, (random() * $max)::bigint::bit($dim) FROM generate_series(1, 10000) i;"
);
# Generate queries
for (1 .. 20)
{
my $r = int(rand() * $max);
push(@queries, "${r}::bigint::bit($dim)");
}
# Check each index type
my @operators = ("<~>", "<\%>");
my @opclasses = ("bit_hamming_ops", "bit_jaccard_ops");
for my $i (0 .. $#operators)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Get exact results
@expected = ();
foreach (@queries)
{
# Handle ties
my $res = $node->safe_psql("postgres", qq(
WITH top AS (
SELECT v $operator $_ AS distance FROM tst ORDER BY v $operator $_ LIMIT $limit
)
SELECT i FROM tst WHERE (v $operator $_) <= (SELECT MAX(distance) FROM top)
));
push(@expected, $res);
}
# Build index serially
$node->safe_psql("postgres", qq(
SET max_parallel_maintenance_workers = 0;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
# Test approximate results
my $min = $operator eq "<\%>" ? 0.95 : 0.98;
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel in memory
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET min_parallel_table_scan_size = 1;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
# Test approximate results
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel on disk
# Set parallel_workers on table to use workers with low maintenance_work_mem
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
ALTER TABLE tst SET (parallel_workers = 2);
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '4MB';
CREATE INDEX idx ON tst USING hnsw (v $opclass);
ALTER TABLE tst RESET (parallel_workers);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
$node->safe_psql("postgres", "DROP INDEX idx;");
}
done_testing();

View File

@@ -1,132 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More;
my $node;
my @queries = ();
my @expected;
my $limit = 20;
my $dim = 10;
my $array_sql = join(",", ('random()') x $dim);
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids;
my @expected_ids = split("\n", $expected[$i]);
foreach (@expected_ids)
{
if (exists($actual_set{$_}))
{
$correct++;
}
$total++;
}
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v halfvec($dim));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
);
# Generate queries
for (1 .. 20)
{
my @r = ();
for (1 .. $dim)
{
push(@r, rand());
}
push(@queries, "[" . join(",", @r) . "]");
}
# Check each index type
my @operators = ("<->", "<#>", "<=>");
my @opclasses = ("halfvec_l2_ops", "halfvec_ip_ops", "halfvec_cosine_ops");
for my $i (0 .. $#operators)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Get exact results
@expected = ();
foreach (@queries)
{
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
push(@expected, $res);
}
# Build index serially
$node->safe_psql("postgres", qq(
SET max_parallel_maintenance_workers = 0;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
# Test approximate results
my $min = $operator eq "<#>" ? 0.95 : 0.98;
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel in memory
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET min_parallel_table_scan_size = 1;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
# Test approximate results
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel on disk
# Set parallel_workers on table to use workers with low maintenance_work_mem
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
ALTER TABLE tst SET (parallel_workers = 2);
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '4MB';
CREATE INDEX idx ON tst USING hnsw (v $opclass);
ALTER TABLE tst RESET (parallel_workers);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
$node->safe_psql("postgres", "DROP INDEX idx;");
}
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.6.2'
default_version = '0.6.0'
module_pathname = '$libdir/vector'
relocatable = true