Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
2867742f3d Test sparsevec vacuum recall 2024-04-23 21:54:15 -07:00
173 changed files with 3570 additions and 4963 deletions

View File

@@ -8,18 +8,8 @@ jobs:
fail-fast: false
matrix:
include:
# - postgres: 18
# os: ubuntu-24.04
- postgres: 17
os: ubuntu-24.04
- postgres: 16
os: ubuntu-22.04
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-20.04
- postgres: 13
os: ubuntu-20.04
os: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
@@ -38,95 +28,4 @@ jobs:
- run: |
sudo apt-get update
sudo apt-get install libipc-run-perl
- run: make prove_installcheck
mac:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 16
os: macos-14
- postgres: 14
os: macos-12
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
# Homebrew Postgres does not enable TAP tests, so need to download
- run: |
brew install cpanm
cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
env:
TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }}
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
- if: ${{ failure() }}
run: cat regression.diffs
i386:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
container:
image: debian:12
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- if: ${{ failure() }}
run: cat pgvector/regression.diffs
valgrind:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 16
check-ub: yes
- run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install
- run: make installcheck
- run: for run in {1..100}; do make prove_installcheck PROVE_TESTS=test/t/031_hnsw_sparsevec_vacuum_recall.pl || break; done

View File

@@ -1,34 +1,4 @@
## 0.8.0 (unreleased)
- Added support for iterative index scans
- Added casts for arrays to `sparsevec`
- Improved cost estimation
- Reduced memory usage for HNSW index scans
- Dropped support for Postgres 12
## 0.7.4 (2024-08-05)
- Fixed locking for parallel HNSW index builds
- Fixed compilation error with GCC 14 on i386 when SSE2 is not enabled
## 0.7.3 (2024-07-22)
- Fixed `failed to add index item` error with `sparsevec`
- Fixed compilation error with FreeBSD ARM
- Fixed compilation warning with MSVC and Postgres 16
## 0.7.2 (2024-06-11)
- Fixed initialization fork for indexes on unlogged tables
## 0.7.1 (2024-06-03)
- Improved performance of on-disk HNSW index builds
- Fixed `undefined symbol` error with GCC 8
- Fixed compilation error with universal binaries on Mac
- Fixed compilation warning with Clang < 14
## 0.7.0 (2024-04-29)
## 0.7.0 (unreleased)
- Added `halfvec` type
- Added `sparsevec` type

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.7.4",
"version": "0.6.2",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.7.4",
"version": "0.6.2",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,9 +1,8 @@
EXTENSION = vector
EXTVERSION = 0.7.4
EXTVERSION = 0.6.2
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)
DATA_built = sql/$(EXTENSION)--$(EXTVERSION).sql
DATA = $(wildcard sql/*--*.sql)
OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
@@ -11,7 +10,6 @@ TESTS = $(wildcard test/sql/*.sql)
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# To compile for portability, run: make OPTFLAGS=""
OPTFLAGS = -march=native
# Mac ARM doesn't always support -march=native
@@ -43,6 +41,8 @@ all: sql/$(EXTENSION)--$(EXTVERSION).sql
sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql
cp $< $@
EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql
PG_CONFIG ?= pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
@@ -52,7 +52,7 @@ ifeq ($(PROVE),)
PROVE = prove
endif
# for Postgres < 15
# for Postgres 15
PROVE_FLAGS += -I ./test/perl
prove_installcheck:

View File

@@ -1,11 +1,10 @@
EXTENSION = vector
EXTVERSION = 0.7.4
EXTVERSION = 0.6.2
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type
REGRESS = bit_functions btree_halfvec btree_sparsevec btree_vector cast copy halfvec_functions halfvec_input hnsw_bit_hamming hnsw_bit_jaccard hnsw_halfvec_cosine hnsw_halfvec_ip hnsw_halfvec_l2 hnsw_options hnsw_sparsevec_cosine hnsw_sparsevec_ip hnsw_sparsevec_l2 hnsw_unlogged hnsw_vector_cosine hnsw_vector_ip hnsw_vector_l2 ivfflat_halfvec_cosine ivfflat_halfvec_ip ivfflat_halfvec_l2 ivfflat_options ivfflat_unlogged ivfflat_vector_cosine ivfflat_vector_ip ivfflat_vector_l2 sparsevec_functions sparsevec_input vector_functions vector_input
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# For /arch flags
@@ -20,6 +19,11 @@ PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
# https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
# PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
all: sql\$(EXTENSION)--$(EXTVERSION).sql
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
# TODO use pg_config
!ifndef PGROOT
!error PGROOT is not set
@@ -39,18 +43,15 @@ SHLIB = $(EXTENSION).dll
LIBS = "$(LIBDIR)\postgres.lib"
all: $(SHLIB) $(DATA_built)
.c.obj:
$(CC) $(CFLAGS) /c $< /Fo$@
$(SHLIB): $(OBJS)
$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
all: $(SHLIB)
install: all
install:
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
@@ -69,6 +70,6 @@ uninstall:
clean:
del /f $(SHLIB) $(EXTENSION).lib $(EXTENSION).exp
del /f $(DATA_built)
del /f $(OBJS)
del /f sql\$(EXTENSION)--$(EXTVERSION).sql
del /f /s /q results regression.diffs regression.out tmp_check tmp_check_iso log output_iso

216
README.md
View File

@@ -5,8 +5,7 @@ Open-source vector similarity search for Postgres
Store your vectors with the rest of your data. Supports:
- exact and approximate nearest neighbor search
- single-precision, half-precision, binary, and sparse vectors
- L2 distance, inner product, cosine distance, L1 distance, Hamming distance, and Jaccard distance
- L2 distance, inner product, and cosine distance
- any [language](#languages) with a Postgres client
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
@@ -21,7 +20,7 @@ Compile and install the extension (supports Postgres 12+)
```sh
cd /tmp
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -46,7 +45,7 @@ Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP%
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -82,7 +81,7 @@ Get the nearest neighbors by L2 distance
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0)
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, unreleased)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
@@ -106,7 +105,7 @@ Insert vectors
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
```
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py))
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py))
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -144,7 +143,7 @@ Supported distance functions are:
- `<->` - L2 distance
- `<#>` - (negative) inner product
- `<=>` - cosine distance
- `<+>` - L1 distance (added in 0.7.0)
- `<+>` - L1 distance (unreleased)
Get the nearest neighbors to a row
@@ -202,7 +201,7 @@ You can add an index to use approximate nearest neighbor search, which trades so
Supported index types are:
- [HNSW](#hnsw)
- [HNSW](#hnsw) - added in 0.5.0
- [IVFFlat](#ivfflat)
## HNSW
@@ -217,8 +216,6 @@ L2 distance
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
```
Note: Use `halfvec_l2_ops` for `halfvec` and `sparsevec_l2_ops` for `sparsevec` (and similar with the other distance functions)
Inner product
```sql
@@ -231,19 +228,19 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
```
L1 distance - added in 0.7.0
L1 distance - unreleased
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
```
Hamming distance - added in 0.7.0
Hamming distance - unreleased
```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
```
Jaccard distance - added in 0.7.0
Jaccard distance - unreleased
```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
@@ -252,9 +249,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions (added in 0.7.0)
- `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0)
- `halfvec` - up to 4,000 dimensions (unreleased)
- `bit` - up to 64,000 dimensions (unreleased)
- `sparsevec` - up to 1,000 non-zero elements (unreleased)
### Index Options
@@ -347,8 +344,6 @@ L2 distance
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
```
Note: Use `halfvec_l2_ops` for `halfvec` (and similar with the other distance functions)
Inner product
```sql
@@ -361,7 +356,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
```
Hamming distance - added in 0.7.0
Hamming distance - unreleased
```sql
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
@@ -370,8 +365,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions (added in 0.7.0)
- `bit` - up to 64,000 dimensions (added in 0.7.0)
- `halfvec` - up to 4,000 dimensions (unreleased)
- `bit` - up to 64,000 dimensions (unreleased)
### Query Options
@@ -445,66 +440,9 @@ Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
```
## Streaming Queries [unreleased]
## Half Vectors
*Added in 0.8.0*
With approximate indexes, you can end up with less results than expected due to filtering conditions in the query.
Starting with 0.8.0, you can enable streaming queries. If too few results from the initial index scan match the query filters, it will resume scanning until enough results are found. This can significantly improve recall (at the cost of speed).
```tsql
SET hnsw.streaming = on;
-- or
SET ivfflat.streaming = on;
```
### Streaming Options
Since scanning a large portion of the index is expensive, there are options to control when the scan ends.
#### HNSW
Specify the max number of additional tuples visited
```sql
SET hnsw.ef_stream = 10000;
```
The scan will also end if reaches `work_mem`, at which point a notice is shown
```text
NOTICE: hnsw index scan exceeded work_mem after 50000 tuples
HINT: Increase work_mem to scan more tuples.
```
Adjust this with:
```sql
SET work_mem = '8MB';
```
#### IVFFlat
Specify the max number of probes
```sql
SET ivfflat.max_probes = 100;
```
### Streaming Order
With streaming queries, its possible for rows to be slightly out of order by distance. For strict ordering, use:
```sql
WITH approx_order AS MATERIALIZED (
SELECT *, embedding <-> '[1,2,3]' AS distance FROM items WHERE ... ORDER BY distance LIMIT 5
) SELECT * FROM approx_order ORDER BY distance;
```
## Half-Precision Vectors
*Added in 0.7.0*
*Unreleased*
Use the `halfvec` type to store half-precision vectors
@@ -512,11 +450,11 @@ Use the `halfvec` type to store half-precision vectors
CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
```
## Half-Precision Indexing
## Half Indexing
*Added in 0.7.0*
*Unreleased*
Index vectors at half precision for smaller indexes
Index vectors at half precision for smaller indexes and faster build times
```sql
CREATE INDEX ON items USING hnsw ((embedding::halfvec(3)) halfvec_l2_ops);
@@ -530,30 +468,30 @@ SELECT * FROM items ORDER BY embedding::halfvec(3) <-> '[1,2,3]' LIMIT 5;
## Binary Vectors
Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/imagehash/example.py))
Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py))
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111');
```
Get the nearest neighbors by Hamming distance (added in 0.7.0)
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Or (before 0.7.0)
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
```
Or (unreleased)
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Also supports Jaccard distance (`<%>`)
## Binary Quantization
*Added in 0.7.0*
*Unreleased*
Use expression indexing for binary quantization
@@ -577,7 +515,7 @@ SELECT * FROM (
## Sparse Vectors
*Added in 0.7.0*
*Unreleased*
Use the `sparsevec` type to store sparse vectors
@@ -608,11 +546,11 @@ SELECT id, content FROM items, plainto_tsquery('hello search') query
WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
```
You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/cross_encoder.py) to combine results.
You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search_rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py) to combine results.
## Indexing Subvectors
*Added in 0.7.0*
*Unreleased*
Use expression indexing to index subvectors
@@ -654,7 +592,7 @@ Be sure to restart Postgres for changes to take effect.
### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py)).
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -744,7 +682,7 @@ Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
## Languages
@@ -788,7 +726,7 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions?
You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
Youll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
#### Can I store vectors with different dimensions in the same column?
@@ -851,7 +789,7 @@ SELECT pg_size_pretty(pg_relation_size('index_name'));
#### Why isnt a query using an index?
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator (not an expression) in ascending order.
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator, not an expression.
```sql
-- index
@@ -926,23 +864,23 @@ Operator | Description | Added
\+ | element-wise addition |
\- | element-wise subtraction |
\* | element-wise multiplication | 0.5.0
\|\| | concatenate | 0.7.0
\|\| | concatenate | unreleased
<-> | Euclidean distance |
<#> | negative inner product |
<=> | cosine distance |
<+> | taxicab distance | 0.7.0
<+> | taxicab distance | unreleased
### Vector Functions
Function | Description | Added
--- | --- | ---
binary_quantize(vector) → bit | binary quantize | 0.7.0
binary_quantize(vector) → bit | binary quantize | unreleased
cosine_distance(vector, vector) → double precision | cosine distance |
inner_product(vector, vector) → double precision | inner product |
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
l2_distance(vector, vector) → double precision | Euclidean distance |
l2_normalize(vector) → vector | Normalize with Euclidean norm | 0.7.0
subvector(vector, integer, integer) → vector | subvector | 0.7.0
l2_normalize(vector) → vector | Normalize with Euclidean norm | unreleased
subvector(vector, integer, integer) → vector | subvector | unreleased
vector_dims(vector) → integer | number of dimensions |
vector_norm(vector) → double precision | Euclidean norm |
@@ -961,35 +899,35 @@ Each half vector takes `2 * dimensions + 8` bytes of storage. Each element is a
Operator | Description | Added
--- | --- | ---
\+ | element-wise addition | 0.7.0
\- | element-wise subtraction | 0.7.0
\* | element-wise multiplication | 0.7.0
\|\| | concatenate | 0.7.0
<-> | Euclidean distance | 0.7.0
<#> | negative inner product | 0.7.0
<=> | cosine distance | 0.7.0
<+> | taxicab distance | 0.7.0
\+ | element-wise addition | unreleased
\- | element-wise subtraction | unreleased
\* | element-wise multiplication | unreleased
\|\| | concatenate | unreleased
<-> | Euclidean distance | unreleased
<#> | negative inner product | unreleased
<=> | cosine distance | unreleased
<+> | taxicab distance | unreleased
### Halfvec Functions
Function | Description | Added
--- | --- | ---
binary_quantize(halfvec) → bit | binary quantize | 0.7.0
cosine_distance(halfvec, halfvec) → double precision | cosine distance | 0.7.0
inner_product(halfvec, halfvec) → double precision | inner product | 0.7.0
l1_distance(halfvec, halfvec) → double precision | taxicab distance | 0.7.0
l2_distance(halfvec, halfvec) → double precision | Euclidean distance | 0.7.0
l2_norm(halfvec) → double precision | Euclidean norm | 0.7.0
l2_normalize(halfvec) → halfvec | Normalize with Euclidean norm | 0.7.0
subvector(halfvec, integer, integer) → halfvec | subvector | 0.7.0
vector_dims(halfvec) → integer | number of dimensions | 0.7.0
binary_quantize(halfvec) → bit | binary quantize | unreleased
cosine_distance(halfvec, halfvec) → double precision | cosine distance | unreleased
inner_product(halfvec, halfvec) → double precision | inner product | unreleased
l1_distance(halfvec, halfvec) → double precision | taxicab distance | unreleased
l2_distance(halfvec, halfvec) → double precision | Euclidean distance | unreleased
l2_norm(halfvec) → double precision | Euclidean norm | unreleased
l2_normalize(halfvec) → halfvec | Normalize with Euclidean norm | unreleased
subvector(halfvec, integer, integer) → halfvec | subvector | unreleased
vector_dims(halfvec) → integer | number of dimensions | unreleased
### Halfvec Aggregate Functions
Function | Description | Added
--- | --- | ---
avg(halfvec) → halfvec | average | 0.7.0
sum(halfvec) → halfvec | sum | 0.7.0
avg(halfvec) → halfvec | average | unreleased
sum(halfvec) → halfvec | sum | unreleased
### Bit Type
@@ -999,15 +937,15 @@ Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres d
Operator | Description | Added
--- | --- | ---
<~> | Hamming distance | 0.7.0
<%> | Jaccard distance | 0.7.0
<~> | Hamming distance | unreleased
<%> | Jaccard distance | unreleased
### Bit Functions
Function | Description | Added
--- | --- | ---
hamming_distance(bit, bit) → double precision | Hamming distance | 0.7.0
jaccard_distance(bit, bit) → double precision | Jaccard distance | 0.7.0
hamming_distance(bit, bit) → double precision | Hamming distance | unreleased
jaccard_distance(bit, bit) → double precision | Jaccard distance | unreleased
### Sparsevec Type
@@ -1017,21 +955,21 @@ Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each ele
Operator | Description | Added
--- | --- | ---
<-> | Euclidean distance | 0.7.0
<#> | negative inner product | 0.7.0
<=> | cosine distance | 0.7.0
<+> | taxicab distance | 0.7.0
<-> | Euclidean distance | unreleased
<#> | negative inner product | unreleased
<=> | cosine distance | unreleased
<+> | taxicab distance | unreleased
### Sparsevec Functions
Function | Description | Added
--- | --- | ---
cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | 0.7.0
inner_product(sparsevec, sparsevec) → double precision | inner product | 0.7.0
l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | 0.7.0
l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | 0.7.0
l2_norm(sparsevec) → double precision | Euclidean norm | 0.7.0
l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | unreleased
inner_product(sparsevec, sparsevec) → double precision | inner product | unreleased
l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | unreleased
l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | unreleased
l2_norm(sparsevec) → double precision | Euclidean norm | unreleased
l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | unreleased
## Installation Notes - Linux and Mac
@@ -1108,9 +1046,9 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually:
```sh
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=16 -t myuser/pgvector .
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
```
### Homebrew

View File

@@ -22,26 +22,52 @@ CREATE OPERATOR || (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_concat
);
CREATE FUNCTION ivfflat_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION ivfflat_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_sparsevec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE OPERATOR CLASS vector_l1_ops
FOR TYPE vector USING hnsw AS
OPERATOR 1 <+> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(vector, vector);
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION bit_ivfflat_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION bit_hnsw_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING ivfflat AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hamming_distance(bit, bit),
FUNCTION 6 bit_ivfflat_support(internal);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 4 bit_hnsw_support(internal);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit),
FUNCTION 4 bit_hnsw_support(internal);
CREATE TYPE halfvec;
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
@@ -143,14 +169,17 @@ CREATE FUNCTION halfvec_accum(double precision[], halfvec) RETURNS double precis
CREATE FUNCTION halfvec_avg(double precision[]) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME', 'vector_combine' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_ivfflat_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_hnsw_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE AGGREGATE avg(halfvec) (
SFUNC = halfvec_accum,
STYPE = double precision[],
FINALFUNC = halfvec_avg,
COMBINEFUNC = halfvec_combine,
COMBINEFUNC = vector_combine,
INITCOND = '{0}',
PARALLEL = SAFE
);
@@ -298,7 +327,7 @@ CREATE OPERATOR CLASS halfvec_l2_ops
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 l2_distance(halfvec, halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
FUNCTION 6 halfvec_ivfflat_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING ivfflat AS
@@ -306,7 +335,8 @@ CREATE OPERATOR CLASS halfvec_ip_ops
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
FUNCTION 5 l2_normalize(halfvec),
FUNCTION 6 halfvec_ivfflat_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING ivfflat AS
@@ -315,67 +345,34 @@ CREATE OPERATOR CLASS halfvec_cosine_ops
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
FUNCTION 5 l2_normalize(halfvec),
FUNCTION 6 halfvec_ivfflat_support(internal);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
FUNCTION 3 l2_normalize(halfvec),
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE OPERATOR CLASS halfvec_l1_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <+> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING ivfflat AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hamming_distance(bit, bit),
FUNCTION 5 ivfflat_bit_support(internal);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE TYPE sparsevec;
@@ -463,6 +460,9 @@ CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparseve
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_hnsw_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (sparsevec AS sparsevec)
WITH FUNCTION sparsevec(sparsevec, integer, boolean) AS IMPLICIT;
@@ -547,23 +547,24 @@ CREATE OPERATOR CLASS sparsevec_l2_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 4 sparsevec_hnsw_support(internal);
CREATE OPERATOR CLASS sparsevec_ip_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 4 sparsevec_hnsw_support(internal);
CREATE OPERATOR CLASS sparsevec_cosine_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 2 l2_norm(sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 3 l2_normalize(sparsevec),
FUNCTION 4 sparsevec_hnsw_support(internal);
CREATE OPERATOR CLASS sparsevec_l1_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 4 sparsevec_hnsw_support(internal);

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.1'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.2'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.4'" to load this file. \quit

View File

@@ -1,26 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.0'" to load this file. \quit
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (integer[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;

View File

@@ -261,23 +261,6 @@ CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnswhandler;
COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
-- access method private functions
CREATE FUNCTION ivfflat_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION ivfflat_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_sparsevec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
-- vector opclasses
CREATE OPERATOR CLASS vector_ops
@@ -331,6 +314,55 @@ CREATE OPERATOR CLASS vector_l1_ops
OPERATOR 1 <+> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(vector, vector);
-- bit functions
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- bit private functions
CREATE FUNCTION bit_ivfflat_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION bit_hnsw_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- bit operators
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
-- bit opclasses
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING ivfflat AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hamming_distance(bit, bit),
FUNCTION 6 bit_ivfflat_support(internal);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 4 bit_hnsw_support(internal);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit),
FUNCTION 4 bit_hnsw_support(internal);
-- halfvec type
CREATE TYPE halfvec;
@@ -438,8 +470,11 @@ CREATE FUNCTION halfvec_accum(double precision[], halfvec) RETURNS double precis
CREATE FUNCTION halfvec_avg(double precision[]) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME', 'vector_combine' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_ivfflat_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_hnsw_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec aggregates
@@ -447,7 +482,7 @@ CREATE AGGREGATE avg(halfvec) (
SFUNC = halfvec_accum,
STYPE = double precision[],
FINALFUNC = halfvec_avg,
COMBINEFUNC = halfvec_combine,
COMBINEFUNC = vector_combine,
INITCOND = '{0}',
PARALLEL = SAFE
);
@@ -603,7 +638,7 @@ CREATE OPERATOR CLASS halfvec_l2_ops
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 l2_distance(halfvec, halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
FUNCTION 6 halfvec_ivfflat_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING ivfflat AS
@@ -611,7 +646,8 @@ CREATE OPERATOR CLASS halfvec_ip_ops
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
FUNCTION 5 l2_normalize(halfvec),
FUNCTION 6 halfvec_ivfflat_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING ivfflat AS
@@ -620,73 +656,34 @@ CREATE OPERATOR CLASS halfvec_cosine_ops
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
FUNCTION 5 l2_normalize(halfvec),
FUNCTION 6 halfvec_ivfflat_support(internal);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
FUNCTION 3 l2_normalize(halfvec),
FUNCTION 4 halfvec_hnsw_support(internal);
CREATE OPERATOR CLASS halfvec_l1_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <+> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
-- bit functions
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- bit operators
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
-- bit opclasses
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING ivfflat AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hamming_distance(bit, bit),
FUNCTION 5 ivfflat_bit_support(internal);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
FUNCTION 4 halfvec_hnsw_support(internal);
--- sparsevec type
@@ -782,16 +779,7 @@ CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparseve
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
CREATE FUNCTION sparsevec_hnsw_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec casts
@@ -811,18 +799,6 @@ CREATE CAST (sparsevec AS halfvec)
CREATE CAST (halfvec AS sparsevec)
WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (integer[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;
-- sparsevec operators
CREATE OPERATOR <-> (
@@ -896,23 +872,24 @@ CREATE OPERATOR CLASS sparsevec_l2_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 4 sparsevec_hnsw_support(internal);
CREATE OPERATOR CLASS sparsevec_ip_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 4 sparsevec_hnsw_support(internal);
CREATE OPERATOR CLASS sparsevec_cosine_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 2 l2_norm(sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 3 l2_normalize(sparsevec),
FUNCTION 4 sparsevec_hnsw_support(internal);
CREATE OPERATOR CLASS sparsevec_l1_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
FUNCTION 4 sparsevec_hnsw_support(internal);

View File

@@ -1,17 +1,12 @@
#include "postgres.h"
#include "bitutils.h"
#include "halfvec.h" /* for USE_DISPATCH and USE_TARGET_CLONES */
#include "port/pg_bitutils.h"
#if defined(USE_DISPATCH)
#define BIT_DISPATCH
#endif
#ifdef BIT_DISPATCH
#include <immintrin.h>
#if defined(USE__GET_CPUID)
#if defined(HAVE__GET_CPUID)
#include <cpuid.h>
#else
#include <intrin.h>
@@ -173,7 +168,7 @@ SupportsAvx512Popcount()
{
unsigned int exx[4] = {0, 0, 0, 0};
#if defined(USE__GET_CPUID)
#if defined(HAVE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#else
__cpuid(exx, 1);
@@ -187,7 +182,7 @@ SupportsAvx512Popcount()
if ((_xgetbv(0) & 0xe6) != 0xe6)
return false;
#if defined(USE__GET_CPUID)
#if defined(HAVE__GET_CPUID)
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]);
#else
__cpuidex(exx, 7, 0);

View File

@@ -1,11 +1,26 @@
#ifndef BITUTILS_H
#define BITUTILS_H
#include "postgres.h"
/* We use two types of dispatching: intrinsics and target_clones */
/* TODO Move to better place */
#ifndef DISABLE_DISPATCH
/* Only enable for more recent compilers to keep build process simple */
#if defined(__x86_64__) && defined(__GNUC__) && __GNUC__ >= 8
#define USE_DISPATCH
#elif defined(__x86_64__) && defined(__clang_major__) && __clang_major__ >= 7
#define USE_DISPATCH
#elif defined(_M_AMD64) && defined(_MSC_VER) && _MSC_VER >= 1920
#define USE_DISPATCH
#endif
#endif
/* Check version in first header */
#if PG_VERSION_NUM < 130000
#error "Requires PostgreSQL 13+"
/* target_clones requires glibc */
#if defined(USE_DISPATCH) && defined(__gnu_linux__)
#define USE_TARGET_CLONES
#endif
#if defined(USE_DISPATCH)
#define BIT_DISPATCH
#endif
extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);

View File

@@ -3,7 +3,6 @@
#include "bitutils.h"
#include "bitvec.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
@@ -41,7 +40,7 @@ CheckDims(VarBit *a, VarBit *b)
/*
* Get the Hamming distance between two bit vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hamming_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
Datum
hamming_distance(PG_FUNCTION_ARGS)
{
@@ -56,7 +55,7 @@ hamming_distance(PG_FUNCTION_ARGS)
/*
* Get the Jaccard distance between two bit vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(jaccard_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(jaccard_distance);
Datum
jaccard_distance(PG_FUNCTION_ARGS)
{

View File

@@ -1,5 +1,5 @@
#ifndef BITVEC_H
#define BITVEC_H
#ifndef BITVECTOR_H
#define BITVECTOR_H
#include "utils/varbit.h"

View File

@@ -6,7 +6,7 @@
#ifdef HALFVEC_DISPATCH
#include <immintrin.h>
#if defined(USE__GET_CPUID)
#if defined(HAVE__GET_CPUID)
#include <cpuid.h>
#else
#include <intrin.h>
@@ -254,7 +254,7 @@ SupportsCpuFeature(unsigned int feature)
{
unsigned int exx[4] = {0, 0, 0, 0};
#if defined(USE__GET_CPUID)
#if defined(HAVE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#else
__cpuid(exx, 1);

View File

@@ -67,6 +67,9 @@ HalfToFloat4(half num)
#elif defined(FLT16_SUPPORT)
return (float) num;
#else
/* TODO Improve performance */
/* Assumes same endianness for floats and integers */
union
{
float f;
@@ -151,6 +154,9 @@ Float4ToHalfUnchecked(float num)
#elif defined(FLT16_SUPPORT)
return (_Float16) num;
#else
/* TODO Improve performance */
/* Assumes same endianness for floats and integers */
union
{
float f;

View File

@@ -19,6 +19,11 @@
#include "utils/numeric.h"
#include "vector.h"
#if PG_VERSION_NUM < 130000
#define TYPALIGN_DOUBLE 'd'
#define TYPALIGN_INT 'i'
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -159,10 +164,28 @@ CheckStateArray(ArrayType *statearray, const char *caller)
return (float8 *) ARR_DATA_PTR(statearray);
}
#if PG_VERSION_NUM < 120003
static pg_noinline void
float_overflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: overflow")));
}
static pg_noinline void
float_underflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: underflow")));
}
#endif
/*
* Convert textual representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_in);
Datum
halfvec_in(PG_FUNCTION_ARGS)
{
@@ -276,7 +299,7 @@ halfvec_in(PG_FUNCTION_ARGS)
/*
* Convert internal representation to textual representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_out);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_out);
Datum
halfvec_out(PG_FUNCTION_ARGS)
{
@@ -305,10 +328,6 @@ halfvec_out(PG_FUNCTION_ARGS)
if (i > 0)
AppendChar(ptr, ',');
/*
* Use shortest decimal representation of single-precision float for
* simplicity
*/
AppendFloat(ptr, HalfToFloat4(vector->x[i]));
}
@@ -322,7 +341,7 @@ halfvec_out(PG_FUNCTION_ARGS)
/*
* Convert type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_typmod_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_typmod_in);
Datum
halfvec_typmod_in(PG_FUNCTION_ARGS)
{
@@ -353,7 +372,7 @@ halfvec_typmod_in(PG_FUNCTION_ARGS)
/*
* Convert external binary representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_recv);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_recv);
Datum
halfvec_recv(PG_FUNCTION_ARGS)
{
@@ -387,7 +406,7 @@ halfvec_recv(PG_FUNCTION_ARGS)
/*
* Convert internal representation to the external binary representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_send);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_send);
Datum
halfvec_send(PG_FUNCTION_ARGS)
{
@@ -407,7 +426,7 @@ halfvec_send(PG_FUNCTION_ARGS)
* Convert half vector to half vector
* This is needed to check the type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec);
Datum
halfvec(PG_FUNCTION_ARGS)
{
@@ -422,7 +441,7 @@ halfvec(PG_FUNCTION_ARGS)
/*
* Convert array to half vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_halfvec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_halfvec);
Datum
array_to_halfvec(PG_FUNCTION_ARGS)
{
@@ -496,7 +515,7 @@ array_to_halfvec(PG_FUNCTION_ARGS)
/*
* Convert half vector to float4[]
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_float4);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_float4);
Datum
halfvec_to_float4(PG_FUNCTION_ARGS)
{
@@ -520,7 +539,7 @@ halfvec_to_float4(PG_FUNCTION_ARGS)
/*
* Convert vector to half vec
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_halfvec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_halfvec);
Datum
vector_to_halfvec(PG_FUNCTION_ARGS)
{
@@ -542,7 +561,7 @@ vector_to_halfvec(PG_FUNCTION_ARGS)
/*
* Get the L2 distance between half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_distance);
Datum
halfvec_l2_distance(PG_FUNCTION_ARGS)
{
@@ -557,7 +576,7 @@ halfvec_l2_distance(PG_FUNCTION_ARGS)
/*
* Get the L2 squared distance between half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_squared_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_squared_distance);
Datum
halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
{
@@ -572,7 +591,7 @@ halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
/*
* Get the inner product of two half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_inner_product);
Datum
halfvec_inner_product(PG_FUNCTION_ARGS)
{
@@ -587,7 +606,7 @@ halfvec_inner_product(PG_FUNCTION_ARGS)
/*
* Get the negative inner product of two half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_negative_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_negative_inner_product);
Datum
halfvec_negative_inner_product(PG_FUNCTION_ARGS)
{
@@ -602,7 +621,7 @@ halfvec_negative_inner_product(PG_FUNCTION_ARGS)
/*
* Get the cosine distance between two half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_cosine_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_cosine_distance);
Datum
halfvec_cosine_distance(PG_FUNCTION_ARGS)
{
@@ -634,7 +653,7 @@ halfvec_cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm)
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_spherical_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_spherical_distance);
Datum
halfvec_spherical_distance(PG_FUNCTION_ARGS)
{
@@ -658,7 +677,7 @@ halfvec_spherical_distance(PG_FUNCTION_ARGS)
/*
* Get the L1 distance between two half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l1_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l1_distance);
Datum
halfvec_l1_distance(PG_FUNCTION_ARGS)
{
@@ -673,7 +692,7 @@ halfvec_l1_distance(PG_FUNCTION_ARGS)
/*
* Get the dimensions of a half vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_vector_dims);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_vector_dims);
Datum
halfvec_vector_dims(PG_FUNCTION_ARGS)
{
@@ -685,7 +704,7 @@ halfvec_vector_dims(PG_FUNCTION_ARGS)
/*
* Get the L2 norm of a half vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_norm);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_norm);
Datum
halfvec_l2_norm(PG_FUNCTION_ARGS)
{
@@ -707,7 +726,7 @@ halfvec_l2_norm(PG_FUNCTION_ARGS)
/*
* Normalize a half vector with the L2 norm
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_normalize);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_normalize);
Datum
halfvec_l2_normalize(PG_FUNCTION_ARGS)
{
@@ -746,7 +765,7 @@ halfvec_l2_normalize(PG_FUNCTION_ARGS)
/*
* Add half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_add);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_add);
Datum
halfvec_add(PG_FUNCTION_ARGS)
{
@@ -785,7 +804,7 @@ halfvec_add(PG_FUNCTION_ARGS)
/*
* Subtract half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_sub);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_sub);
Datum
halfvec_sub(PG_FUNCTION_ARGS)
{
@@ -824,7 +843,7 @@ halfvec_sub(PG_FUNCTION_ARGS)
/*
* Multiply half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_mul);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_mul);
Datum
halfvec_mul(PG_FUNCTION_ARGS)
{
@@ -866,7 +885,7 @@ halfvec_mul(PG_FUNCTION_ARGS)
/*
* Concatenate half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_concat);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_concat);
Datum
halfvec_concat(PG_FUNCTION_ARGS)
{
@@ -890,7 +909,7 @@ halfvec_concat(PG_FUNCTION_ARGS)
/*
* Quantize a half vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_binary_quantize);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_binary_quantize);
Datum
halfvec_binary_quantize(PG_FUNCTION_ARGS)
{
@@ -908,39 +927,24 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
/*
* Get a subvector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_subvector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_subvector);
Datum
halfvec_subvector(PG_FUNCTION_ARGS)
{
HalfVector *a = PG_GETARG_HALFVEC_P(0);
int32 start = PG_GETARG_INT32(1);
int32 count = PG_GETARG_INT32(2);
int32 end;
int32 end = start + count;
half *ax = a->x;
HalfVector *result;
int32 dim;
if (count < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("halfvec must have at least 1 dimension")));
/*
* Check if (start + count > a->dim), avoiding integer overflow. a->dim
* and count are both positive, so a->dim - count won't overflow.
*/
if (start > a->dim - count)
end = a->dim + 1;
else
end = start + count;
int dim;
/* Indexing starts at 1, like substring */
if (start < 1)
start = 1;
else if (start > a->dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("halfvec must have at least 1 dimension")));
if (end > a->dim)
end = a->dim + 1;
dim = end - start;
CheckDim(dim);
@@ -955,7 +959,7 @@ halfvec_subvector(PG_FUNCTION_ARGS)
/*
* Internal helper to compare half vectors
*/
static int
int
halfvec_cmp_internal(HalfVector * a, HalfVector * b)
{
int dim = Min(a->dim, b->dim);
@@ -982,7 +986,7 @@ halfvec_cmp_internal(HalfVector * a, HalfVector * b)
/*
* Less than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_lt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_lt);
Datum
halfvec_lt(PG_FUNCTION_ARGS)
{
@@ -995,7 +999,7 @@ halfvec_lt(PG_FUNCTION_ARGS)
/*
* Less than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_le);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_le);
Datum
halfvec_le(PG_FUNCTION_ARGS)
{
@@ -1008,7 +1012,7 @@ halfvec_le(PG_FUNCTION_ARGS)
/*
* Equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_eq);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_eq);
Datum
halfvec_eq(PG_FUNCTION_ARGS)
{
@@ -1021,7 +1025,7 @@ halfvec_eq(PG_FUNCTION_ARGS)
/*
* Not equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_ne);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ne);
Datum
halfvec_ne(PG_FUNCTION_ARGS)
{
@@ -1034,7 +1038,7 @@ halfvec_ne(PG_FUNCTION_ARGS)
/*
* Greater than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_ge);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ge);
Datum
halfvec_ge(PG_FUNCTION_ARGS)
{
@@ -1047,7 +1051,7 @@ halfvec_ge(PG_FUNCTION_ARGS)
/*
* Greater than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_gt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_gt);
Datum
halfvec_gt(PG_FUNCTION_ARGS)
{
@@ -1060,7 +1064,7 @@ halfvec_gt(PG_FUNCTION_ARGS)
/*
* Compare half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_cmp);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_cmp);
Datum
halfvec_cmp(PG_FUNCTION_ARGS)
{
@@ -1073,7 +1077,7 @@ halfvec_cmp(PG_FUNCTION_ARGS)
/*
* Accumulate half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_accum);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_accum);
Datum
halfvec_accum(PG_FUNCTION_ARGS)
{
@@ -1134,7 +1138,7 @@ halfvec_accum(PG_FUNCTION_ARGS)
/*
* Average half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_avg);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_avg);
Datum
halfvec_avg(PG_FUNCTION_ARGS)
{
@@ -1168,7 +1172,7 @@ halfvec_avg(PG_FUNCTION_ARGS)
/*
* Convert sparse vector to half vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_halfvec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_halfvec);
Datum
sparsevec_to_halfvec(PG_FUNCTION_ARGS)
{
@@ -1183,7 +1187,7 @@ sparsevec_to_halfvec(PG_FUNCTION_ARGS)
result = InitHalfVector(dim);
for (int i = 0; i < svec->nnz; i++)
result->x[svec->indices[i]] = Float4ToHalf(values[i]);
result->x[svec->indices[i] - 1] = Float4ToHalf(values[i]);
PG_RETURN_POINTER(result);
}

View File

@@ -5,31 +5,9 @@
#include <float.h>
/* We use two types of dispatching: intrinsics and target_clones */
/* TODO Move to better place */
#ifndef DISABLE_DISPATCH
/* Only enable for more recent compilers to keep build process simple */
#if defined(__x86_64__) && defined(__GNUC__) && __GNUC__ >= 9
#define USE_DISPATCH
#elif defined(__x86_64__) && defined(__clang_major__) && __clang_major__ >= 7
#define USE_DISPATCH
#elif defined(_M_AMD64) && defined(_MSC_VER) && _MSC_VER >= 1920
#define USE_DISPATCH
#endif
#endif
/* target_clones requires glibc */
#if defined(USE_DISPATCH) && defined(__gnu_linux__) && defined(__has_attribute)
/* Use separate line for portability */
#if __has_attribute(target_clones)
#define USE_TARGET_CLONES
#endif
#endif
/* Apple clang check needed for universal binaries on Mac */
#if defined(USE_DISPATCH) && (defined(HAVE__GET_CPUID) || defined(__apple_build_version__))
#define USE__GET_CPUID
#endif
#include "bitutils.h"
#include "fmgr.h"
#include "vector.h"
#if defined(USE_DISPATCH)
#define HALFVEC_DISPATCH
@@ -38,7 +16,7 @@
/* F16C has better performance than _Float16 (on x86-64) */
#if defined(__F16C__)
#define F16C_SUPPORT
#elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH) && !defined(__FreeBSD__) && (!defined(__i386__) || defined(__SSE2__))
#elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH)
#define FLT16_SUPPORT
#endif
@@ -50,7 +28,7 @@
#define HALF_MAX 65504
#endif
#define HALFVEC_MAX_DIM 16000
#define HALFVEC_MAX_DIM VECTOR_MAX_DIM
#define HALFVEC_SIZE(_dim) (offsetof(HalfVector, x) + sizeof(half)*(_dim))
#define DatumGetHalfVector(x) ((HalfVector *) PG_DETOAST_DATUM(x))
@@ -61,10 +39,11 @@ typedef struct HalfVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int16 dim; /* number of dimensions */
int16 unused; /* reserved for future use, always zero */
int16 unused;
half x[FLEXIBLE_ARRAY_MEMBER];
} HalfVector;
HalfVector *InitHalfVector(int dim);
int halfvec_cmp_internal(HalfVector * a, HalfVector * b);
#endif

View File

@@ -9,18 +9,14 @@
#include "commands/vacuum.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
#endif
int hnsw_ef_search;
int hnsw_ef_stream;
bool hnsw_streaming;
int hnsw_lock_tranche_id;
static relopt_kind hnsw_relopt_kind;
@@ -63,25 +59,22 @@ HnswInit(void)
hnsw_relopt_kind = add_reloption_kind();
add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",
HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M, AccessExclusiveLock);
HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
add_int_reloption(hnsw_relopt_kind, "ef_construction", "Size of the dynamic candidate list for construction",
HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION, AccessExclusiveLock);
HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
"Valid range is 1..1000.", &hnsw_ef_search,
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
/* TODO Figure out name */
DefineCustomBoolVariable("hnsw.streaming", "Use streaming mode",
NULL, &hnsw_streaming,
HNSW_DEFAULT_STREAMING, PGC_USERSET, 0, NULL, NULL, NULL);
/* TODO Figure out name */
/* TODO Use same value as ivfflat.max_probes for "all" */
DefineCustomIntVariable("hnsw.ef_stream", "Sets the max number of additional candidates to visit for streaming search",
"-1 means all", &hnsw_ef_stream,
HNSW_DEFAULT_EF_STREAM, HNSW_MIN_EF_STREAM, HNSW_MAX_EF_STREAM, PGC_USERSET, 0, NULL, NULL, NULL);
MarkGUCPrefixReserved("hnsw");
}
@@ -102,33 +95,6 @@ hnswbuildphasename(int64 phasenum)
}
}
/*
* Estimate ef needed for iterative scans
*/
static int
EstimateEf(PlannerInfo *root, IndexPath *path)
{
double selectivity = 1;
ListCell *lc;
/* Cannot estimate without limit */
/* limit_tuples includes offset */
if (root->limit_tuples < 0)
return 0;
/* Get the selectivity of non-index conditions */
foreach(lc, path->indexinfo->indrestrictinfo)
{
RestrictInfo *rinfo = lfirst(lc);
/* Skip DEFAULT_INEQ_SEL since it may be a distance filter */
if (rinfo->norm_selec >= 0 && rinfo->norm_selec <= 1 && rinfo->norm_selec != (Selectivity) DEFAULT_INEQ_SEL)
selectivity *= rinfo->norm_selec;
}
return root->limit_tuples / Max(selectivity, 0.00001);
}
/*
* Estimate the cost of an index scan
*/
@@ -140,19 +106,14 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
{
GenericCosts costs;
int m;
int ef;
int entryLevel;
int layer0TuplesMax;
double layer0Selectivity;
double scalingFactor = 0.55;
double spc_seq_page_cost;
Relation index;
/* Never use index without order */
if (path->indexorderbys == NULL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
*indexStartupCost = DBL_MAX;
*indexTotalCost = DBL_MAX;
*indexSelectivity = 0;
*indexCorrelation = 0;
*indexPages = 0;
@@ -165,57 +126,15 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
HnswGetMetaPageInfo(index, &m, NULL);
index_close(index, NoLock);
ef = hnsw_streaming ? Max(hnsw_ef_search, EstimateEf(root, path)) : hnsw_ef_search;
/* Approximate entry level */
entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
/*
* HNSW cost estimation follows a formula that accounts for the total
* number of tuples indexed combined with the parameters that most
* influence the duration of the index scan, namely: m - the number of
* tuples that are scanned in each step of the HNSW graph traversal
* ef_search - which influences the total number of steps taken at layer 0
*
* The source of the vector data can impact how many steps it takes to
* converge on the set of vectors to return to the executor. Currently, we
* use a hardcoded scaling factor (HNSWScanScalingFactor) to help
* influence that, but this could later become a configurable parameter
* based on the cost estimations.
*
* The tuple estimator formula is below:
*
* numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
*
* "entryLevel * m" represents the floor of tuples we need to scan to get
* to layer 0 (L0).
*
* "layer0TuplesMax" is the estimated total number of tuples we'd scan at
* L0 if we weren't discarding already visited tuples as part of the scan.
*
* "layer0Selectivity" estimates the percentage of tuples that are scanned
* at L0, accounting for previously visited tuples, multiplied by the
* "scalingFactor" (currently hardcoded).
*/
entryLevel = (int) (log(path->indexinfo->tuples + 1) * HnswGetMl(m));
layer0TuplesMax = HnswGetLayerM(m, 0) * ef;
layer0Selectivity = (scalingFactor * log(path->indexinfo->tuples + 1)) /
(log(m) * (1 + log(ef)));
costs.numIndexTuples = (entryLevel * m) +
(layer0TuplesMax * layer0Selectivity);
/* TODO Improve estimate of visited tuples (currently underestimates) */
/* Account for number of tuples (or entry level), m, and ef_search */
costs.numIndexTuples = (entryLevel + 2) * m;
genericcostestimate(root, path, loop_count, &costs);
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
/* Adjust cost if needed since TOAST not included in seq scan cost */
if (costs.numIndexPages > path->indexinfo->rel->pages)
{
/* Change all page cost from random to sequential */
costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Remove cost of extra pages */
costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
}
/* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost;
*indexTotalCost = costs.indexTotalCost;
@@ -235,10 +154,23 @@ hnswoptions(Datum reloptions, bool validate)
{"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)},
};
#if PG_VERSION_NUM >= 130000
return (bytea *) build_reloptions(reloptions, validate,
hnsw_relopt_kind,
sizeof(HnswOptions),
tab, lengthof(tab));
#else
relopt_value *options;
int numoptions;
HnswOptions *rdopts;
options = parseRelOptions(reloptions, validate, hnsw_relopt_kind, &numoptions);
rdopts = allocateReloptStruct(sizeof(HnswOptions), options, numoptions);
fillRelOptions((void *) rdopts, sizeof(HnswOptions), options, numoptions,
validate, tab, lengthof(tab));
return (bytea *) rdopts;
#endif
}
/*
@@ -255,15 +187,17 @@ hnswvalidate(Oid opclassoid)
*
* See https://www.postgresql.org/docs/current/index-api.html
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
PGDLLEXPORT PG_FUNCTION_INFO_V1(hnswhandler);
Datum
hnswhandler(PG_FUNCTION_ARGS)
{
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
amroutine->amsupport = 3;
amroutine->amsupport = 4;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0;
#endif
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
amroutine->amcanbackward = false; /* can change direction mid-scan */
@@ -276,24 +210,17 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amclusterable = false;
amroutine->ampredlocks = false;
amroutine->amcanparallel = false;
#if PG_VERSION_NUM >= 170000
amroutine->amcanbuildparallel = true;
#endif
amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
#if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false;
#endif
amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
#endif
amroutine->amkeytype = InvalidOid;
/* Interface functions */
amroutine->ambuild = hnswbuild;
amroutine->ambuildempty = hnswbuildempty;
amroutine->aminsert = hnswinsert;
#if PG_VERSION_NUM >= 170000
amroutine->aminsertcleanup = NULL;
#endif
amroutine->ambulkdelete = hnswbulkdelete;
amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL;

View File

@@ -12,8 +12,8 @@
#include "utils/sampling.h"
#include "vector.h"
#ifdef HNSW_BENCH
#include "portability/instr_time.h"
#if PG_VERSION_NUM < 120000
#error "Requires PostgreSQL 12+"
#endif
#define HNSW_MAX_DIM 2000
@@ -22,7 +22,8 @@
/* Support functions */
#define HNSW_DISTANCE_PROC 1
#define HNSW_NORM_PROC 2
#define HNSW_TYPE_INFO_PROC 3
#define HNSW_NORMALIZE_PROC 3
#define HNSW_TYPE_SUPPORT_PROC 4
#define HNSW_VERSION 1
#define HNSW_MAGIC_NUMBER 0xA953A953
@@ -46,10 +47,6 @@
#define HNSW_DEFAULT_EF_SEARCH 40
#define HNSW_MIN_EF_SEARCH 1
#define HNSW_MAX_EF_SEARCH 1000
#define HNSW_DEFAULT_STREAMING false
#define HNSW_DEFAULT_EF_STREAM -1
#define HNSW_MIN_EF_STREAM -1
#define HNSW_MAX_EF_STREAM INT_MAX
/* Tuple types */
#define HNSW_ELEMENT_TUPLE_TYPE 1
@@ -61,6 +58,15 @@
#define HNSW_UPDATE_ENTRY_GREATER 1
#define HNSW_UPDATE_ENTRY_ALWAYS 2
typedef enum HnswType
{
HNSW_TYPE_VECTOR,
HNSW_TYPE_HALFVEC,
HNSW_TYPE_BIT,
HNSW_TYPE_SPARSEVEC,
HNSW_TYPE_UNSUPPORTED
} HnswType;
/* Build phases */
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
#define PROGRESS_HNSW_PHASE_LOAD 2
@@ -76,21 +82,6 @@
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
#ifdef HNSW_BENCH
#define HnswBench(name, code) \
do { \
instr_time start; \
instr_time duration; \
INSTR_TIME_SET_CURRENT(start); \
(code); \
INSTR_TIME_SET_CURRENT(duration); \
INSTR_TIME_SUBTRACT(duration, start); \
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
} while (0)
#else
#define HnswBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -99,6 +90,11 @@
#define SeedRandom(seed) srandom(seed)
#endif
#if PG_VERSION_NUM < 130000
#define list_delete_last(list) list_truncate(list, list_length(list) - 1)
#define list_sort(list, cmp) ((list) = list_qsort(list, cmp))
#endif
#define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
#define HnswIsNeighborTuple(tup) ((tup)->type == HNSW_NEIGHBOR_TUPLE_TYPE)
@@ -129,8 +125,6 @@
/* Variables */
extern int hnsw_ef_search;
extern int hnsw_ef_stream;
extern bool hnsw_streaming;
extern int hnsw_lock_tranche_id;
typedef struct HnswElementData HnswElementData;
@@ -154,7 +148,6 @@ struct HnswElementData
uint8 heaptidsLength;
uint8 level;
uint8 deleted;
uint8 version;
uint32 hash;
HnswNeighborsPtr neighbors;
BlockNumber blkno;
@@ -181,16 +174,11 @@ struct HnswNeighborArray
HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
};
typedef struct HnswSearchCandidate
typedef struct HnswPairingHeapNode
{
pairingheap_node c_node;
pairingheap_node w_node;
HnswElementPtr element;
float distance;
} HnswSearchCandidate;
#define HnswGetSearchCandidate(membername, ptr) pairingheap_container(HnswSearchCandidate, membername, ptr)
#define HnswGetSearchCandidateConst(membername, ptr) pairingheap_const_container(HnswSearchCandidate, membername, ptr)
pairingheap_node ph_node;
HnswCandidate *inner;
} HnswPairingHeapNode;
/* HNSW index options */
typedef struct HnswOptions
@@ -259,13 +247,6 @@ typedef struct HnswAllocator
void *state;
} HnswAllocator;
typedef struct HnswTypeInfo
{
int maxDimensions;
Datum (*normalize) (PG_FUNCTION_ARGS);
void (*checkValue) (Pointer v);
} HnswTypeInfo;
typedef struct HnswBuildState
{
/* Info */
@@ -273,7 +254,7 @@ typedef struct HnswBuildState
Relation index;
IndexInfo *indexInfo;
ForkNumber forkNum;
const HnswTypeInfo *typeInfo;
HnswType type;
/* Settings */
int dimensions;
@@ -287,6 +268,7 @@ typedef struct HnswBuildState
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
/* Variables */
@@ -335,10 +317,10 @@ typedef struct HnswElementTupleData
uint8 type;
uint8 level;
uint8 deleted;
uint8 version;
uint8 unused;
ItemPointerData heaptids[HNSW_HEAPTIDS];
ItemPointerData neighbortid;
uint16 unused;
uint16 unused2;
Vector data;
} HnswElementTupleData;
@@ -347,36 +329,23 @@ typedef HnswElementTupleData * HnswElementTuple;
typedef struct HnswNeighborTupleData
{
uint8 type;
uint8 version;
uint8 unused;
uint16 count;
ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
} HnswNeighborTupleData;
typedef HnswNeighborTupleData * HnswNeighborTuple;
typedef union
{
struct pointerhash_hash *pointers;
struct offsethash_hash *offsets;
struct tidhash_hash *tids;
} visited_hash;
typedef struct HnswScanOpaqueData
{
const HnswTypeInfo *typeInfo;
bool first;
List *w;
visited_hash v;
pairingheap *discarded;
Datum q;
int m;
int64 tuples;
double previousDistance;
MemoryContext tmpCtx;
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
} HnswScanOpaqueData;
@@ -412,19 +381,21 @@ typedef struct HnswVacuumState
int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
Datum HnswNormValue(const HnswTypeInfo * typeInfo, Oid collation, Datum value);
HnswType HnswGetType(Relation index);
Datum HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum value);
bool HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
void HnswCheckValue(Datum value, HnswType type);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page);
void HnswInit(void);
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
@@ -432,12 +403,11 @@ void HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator *
bool HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
void HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance);
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
void HnswLoadNeighbors(HnswElement element, Relation index, int m);
void HnswInitLockTranche(void);
const HnswTypeInfo *HnswGetTypeInfo(Relation index);
PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
/* Index access methods */

View File

@@ -60,6 +60,12 @@
#include "pgstat.h"
#endif
#if PG_VERSION_NUM >= 130000
#define CALLBACK_ITEM_POINTER ItemPointer tid
#else
#define CALLBACK_ITEM_POINTER HeapTuple hup
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_status.h"
#include "utils/wait_event.h"
@@ -69,6 +75,10 @@
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
#if PG_VERSION_NUM < 130000
#define GENERATIONCHUNK_RAWSIZE (SIZEOF_SIZE_T + SIZEOF_VOID_P * 2)
#endif
/*
* Create the metapage
*/
@@ -182,9 +192,7 @@ CreateGraphPages(HnswBuildState * buildstate)
/* Initial size check */
if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("index tuple too large")));
elog(ERROR, "index tuple too large");
HnswSetElementTuple(base, etup, element);
@@ -371,13 +379,7 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
for (int lc = e->level; lc >= 0; lc--)
{
int lm = HnswGetLayerM(m, lc);
Size neighborsSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm);
HnswNeighborArray *neighbors = palloc(neighborsSize);
/* Copy neighbors to local memory */
LWLockAcquire(&e->lock, LW_SHARED);
memcpy(neighbors, HnswGetNeighbors(base, e, lc), neighborsSize);
LWLockRelease(&e->lock);
HnswNeighborArray *neighbors = HnswGetNeighbors(base, e, lc);
for (int i = 0; i < neighbors->length; i++)
{
@@ -387,6 +389,7 @@ UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswEleme
/* Keep scan-build happy on Mac x86-64 */
Assert(neighborElement);
/* Use element for lock instead of hc since hc can be replaced */
LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
LWLockRelease(&neighborElement->lock);
@@ -473,7 +476,6 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
static bool
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, HnswBuildState * buildstate)
{
const HnswTypeInfo *typeInfo = buildstate->typeInfo;
HnswGraph *graph = buildstate->graph;
HnswElement element;
HnswAllocator *allocator = &buildstate->allocator;
@@ -486,8 +488,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Check value */
if (typeInfo->checkValue != NULL)
typeInfo->checkValue(DatumGetPointer(value));
HnswCheckValue(value, buildstate->type);
/* Normalize if needed */
if (buildstate->normprocinfo != NULL)
@@ -495,7 +496,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
if (!HnswCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
return false;
value = HnswNormValue(typeInfo, buildstate->collation, value);
value = HnswNormValue(buildstate->normalizeprocinfo, buildstate->collation, value);
}
/* Get datum size */
@@ -575,13 +576,17 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Callback for table_index_build_scan
*/
static void
BuildCallback(Relation index, ItemPointer tid, Datum *values,
BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state)
{
HnswBuildState *buildstate = (HnswBuildState *) state;
HnswGraph *graph = buildstate->graph;
MemoryContext oldCtx;
#if PG_VERSION_NUM < 130000
ItemPointer tid = &hup->t_self;
#endif
/* Skip nulls */
if (isnull[0])
return;
@@ -644,7 +649,11 @@ HnswMemoryContextAlloc(Size size, void *state)
HnswBuildState *buildstate = (HnswBuildState *) state;
void *chunk = MemoryContextAlloc(buildstate->graphCtx, size);
#if PG_VERSION_NUM >= 130000
buildstate->graphData.memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->graphData.memoryUsed += MAXALIGN(size);
#endif
return chunk;
}
@@ -662,43 +671,53 @@ HnswSharedMemoryAlloc(Size size, void *state)
return chunk;
}
/*
* Get max dimensions
*/
static int
GetMaxDimensions(HnswType type)
{
int maxDimensions = HNSW_MAX_DIM;
if (type == HNSW_TYPE_HALFVEC)
maxDimensions *= 2;
else if (type == HNSW_TYPE_BIT)
maxDimensions *= 32;
else if (type == HNSW_TYPE_SPARSEVEC)
maxDimensions = INT_MAX;
return maxDimensions;
}
/*
* Initialize the build state
*/
static void
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
{
int maxDimensions;
buildstate->heap = heap;
buildstate->index = index;
buildstate->indexInfo = indexInfo;
buildstate->forkNum = forkNum;
buildstate->typeInfo = HnswGetTypeInfo(index);
buildstate->type = HnswGetType(index);
buildstate->m = HnswGetM(index);
buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
/* Disallow varbit since require fixed dimensions */
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("type not supported for hnsw index")));
maxDimensions = GetMaxDimensions(buildstate->type);
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("column does not have dimensions")));
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("column cannot have more than %d dimensions for hnsw index", buildstate->typeInfo->maxDimensions)));
if (buildstate->dimensions > maxDimensions)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", maxDimensions);
if (buildstate->efConstruction < 2 * buildstate->m)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("ef_construction must be greater than or equal to 2 * m")));
elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
buildstate->reltuples = 0;
buildstate->indtuples = 0;
@@ -706,6 +725,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Get support functions */
buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
buildstate->normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
buildstate->collation = index->rd_indcollation[0];
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * 1024L);
@@ -1118,8 +1138,8 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
BuildGraph(buildstate, forkNum);
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
if (RelationNeedsWAL(index))
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocks(index), true);
FreeBuildState(buildstate);
}

View File

@@ -36,15 +36,14 @@ GetInsertPage(Relation index)
* Check for a free offset
*/
static bool
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage, uint8 *tupleVersion)
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
{
OffsetNumber offno;
OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
ItemId eitemid = PageGetItemId(page, offno);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, eitemid);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
@@ -55,9 +54,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
BlockNumber elementPage = BufferGetBlockNumber(buf);
BlockNumber neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
OffsetNumber neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
ItemId nitemid;
Size pageFree;
Size npageFree;
ItemId itemid;
if (!BlockNumberIsValid(*newInsertPage))
*newInsertPage = elementPage;
@@ -76,29 +73,13 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
*npage = BufferGetPage(*nbuf);
}
nitemid = PageGetItemId(*npage, neighborOffno);
itemid = PageGetItemId(*npage, neighborOffno);
/* Ensure aligned for space check */
Assert(etupSize == MAXALIGN(etupSize));
Assert(ntupSize == MAXALIGN(ntupSize));
/*
* Calculate free space individually since tuples are overwritten
* individually (in separate calls to PageIndexTupleOverwrite)
*/
pageFree = ItemIdGetLength(eitemid) + PageGetExactFreeSpace(page);
npageFree = ItemIdGetLength(nitemid);
if (neighborPage != elementPage)
npageFree += PageGetExactFreeSpace(*npage);
else if (pageFree >= etupSize)
npageFree += pageFree - etupSize;
/* Check for space */
if (pageFree >= etupSize && npageFree >= ntupSize)
/* Check for space on neighbor tuple page */
if (PageGetFreeSpace(*npage) + ItemIdGetLength(itemid) - sizeof(ItemIdData) >= ntupSize)
{
*freeOffno = offno;
*freeNeighborOffno = neighborOffno;
*tupleVersion = etup->version;
return true;
}
else if (*nbuf != buf)
@@ -154,7 +135,6 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
OffsetNumber freeOffno = InvalidOffsetNumber;
OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
BlockNumber newInsertPage = InvalidBlockNumber;
uint8 tupleVersion;
char *base = NULL;
/* Calculate sizes */
@@ -204,7 +184,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
}
/* Next, try space from a deleted element */
if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage, &tupleVersion))
if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
{
if (nbuf != buf)
{
@@ -214,10 +194,6 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
npage = GenericXLogRegisterBuffer(state, nbuf, 0);
}
/* Set tuple version */
etup->version = tupleVersion;
ntup->version = tupleVersion;
break;
}
@@ -385,12 +361,8 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, Hns
HnswElement neighborElement = HnswPtrAccess(base, hc->element);
OffsetNumber offno = neighborElement->neighborOffno;
/*
* Get latest neighbors since they may have changed. Do not lock
* yet since selecting neighbors can take time. Could use
* optimistic locking to retry if another update occurs before
* getting exclusive lock.
*/
/* Get latest neighbors since they may have changed */
/* Do not lock yet since selecting neighbors can take time */
HnswLoadNeighbors(neighborElement, index, m);
/*
@@ -640,7 +612,6 @@ static void
HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
{
Datum value;
const HnswTypeInfo *typeInfo = HnswGetTypeInfo(index);
FmgrInfo *normprocinfo;
Oid collation = index->rd_indcollation[0];
@@ -648,8 +619,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Check value */
if (typeInfo->checkValue != NULL)
typeInfo->checkValue(DatumGetPointer(value));
HnswCheckValue(value, HnswGetType(index));
/* Normalize if needed */
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
@@ -658,7 +628,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
if (!HnswCheckNorm(normprocinfo, collation, value))
return;
value = HnswNormValue(typeInfo, collation, value);
value = HnswNormValue(HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC), collation, value);
}
HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false);

View File

@@ -1,7 +1,5 @@
#include "postgres.h"
#include <float.h>
#include "access/relscan.h"
#include "hnsw.h"
#include "pgstat.h"
@@ -28,9 +26,6 @@ GetScanItems(IndexScanDesc scan, Datum q)
/* Get m and entry point */
HnswGetMetaPageInfo(index, &m, &entryPoint);
so->q = q;
so->m = m;
if (entryPoint == NULL)
return NIL;
@@ -38,44 +33,11 @@ GetScanItems(IndexScanDesc scan, Datum q)
for (int lc = entryPoint->level; lc >= 1; lc--)
{
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL, NULL, NULL, true, NULL);
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
ep = w;
}
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, &so->v, hnsw_streaming ? &so->discarded : NULL, true, &so->tuples);
}
/*
* Resume scan at ground level with discarded candidates
*/
static List *
ResumeScanItems(IndexScanDesc scan)
{
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
Relation index = scan->indexRelation;
FmgrInfo *procinfo = so->procinfo;
Oid collation = so->collation;
List *ep = NIL;
char *base = NULL;
int batch_size = hnsw_ef_search;
if (pairingheap_is_empty(so->discarded))
return NIL;
/* Get next batch of candidates */
for (int i = 0; i < batch_size; i++)
{
HnswSearchCandidate *hc;
if (pairingheap_is_empty(so->discarded))
break;
hc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded));
ep = lappend(ep, hc);
}
return HnswSearchLayer(base, so->q, ep, batch_size, 0, index, procinfo, collation, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
}
/*
@@ -97,9 +59,9 @@ GetScanValue(IndexScanDesc scan)
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Normalize if needed */
/* Fine if normalization fails */
if (so->normprocinfo != NULL)
value = HnswNormValue(so->typeInfo, so->collation, value);
value = HnswNormValue(so->normalizeprocinfo, so->collation, value);
}
return value;
@@ -117,10 +79,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
scan = RelationGetIndexScan(index, nkeys, norderbys);
so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
so->typeInfo = HnswGetTypeInfo(index);
so->first = true;
so->v.tids = NULL;
so->discarded = NULL;
so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Hnsw scan temporary context",
ALLOCSET_DEFAULT_SIZES);
@@ -128,6 +87,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
/* Set support functions */
so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
so->normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
so->collation = index->rd_indcollation[0];
scan->opaque = so;
@@ -143,15 +103,7 @@ hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int no
{
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
if (so->v.tids != NULL)
tidhash_reset(so->v.tids);
if (so->discarded != NULL)
pairingheap_reset(so->discarded);
so->first = true;
so->tuples = 0;
so->previousDistance = -INFINITY;
MemoryContextReset(so->tmpCtx);
if (keys && scan->numberOfKeys > 0)
@@ -201,109 +153,34 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
*/
LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
HnswBench("scan iteration", so->w = GetScanItems(scan, value));
so->w = GetScanItems(scan, value);
/* Release shared lock */
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
so->first = false;
#if defined(HNSW_MEMORY)
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
#if defined(HNSW_MEMORY) && PG_VERSION_NUM >= 130000
elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(so->tmpCtx, false) / (1024 * 1024));
#endif
}
for (;;)
while (list_length(so->w) > 0)
{
char *base = NULL;
HnswSearchCandidate *hc;
HnswElement element;
HnswCandidate *hc = llast(so->w);
HnswElement element = HnswPtrAccess(base, hc->element);
ItemPointer heaptid;
if (list_length(so->w) == 0)
{
if (!hnsw_streaming)
break;
/* Empty index */
if (so->discarded == NULL)
break;
/* Reached max number of additional tuples */
if (hnsw_ef_stream != -1 && so->tuples >= hnsw_ef_search + hnsw_ef_stream)
{
if (pairingheap_is_empty(so->discarded))
break;
/* Return remaining tuples */
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
}
/* Prevent scans from consuming too much memory */
else if (MemoryContextMemAllocated(so->tmpCtx, false) > (Size) work_mem * 1024L)
{
if (pairingheap_is_empty(so->discarded))
{
ereport(NOTICE,
(errmsg("hnsw index scan exceeded work_mem after " INT64_FORMAT " tuples", so->tuples),
errhint("Increase work_mem to scan more tuples.")));
break;
}
/* Return remaining tuples */
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
}
else
{
/*
* Locking ensures when neighbors are read, the elements they
* reference will not be deleted (and replaced) during the
* iteration.
*
* Elements loaded into memory on previous iterations may have
* been deleted (and replaced), so when reading neighbors, the
* element version must be checked.
*/
LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
HnswBench("scan iteration", so->w = ResumeScanItems(scan));
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
#if defined(HNSW_MEMORY)
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
#endif
}
if (list_length(so->w) == 0)
break;
}
hc = llast(so->w);
element = HnswPtrAccess(base, hc->element);
/* Move to next element if no valid heap TIDs */
if (element->heaptidsLength == 0)
{
so->w = list_delete_last(so->w);
/* Mark memory as free for next iteration */
if (hnsw_streaming)
{
pfree(element);
pfree(hc);
}
continue;
}
heaptid = &element->heaptids[--element->heaptidsLength];
if (hc->distance < so->previousDistance)
continue;
so->previousDistance = hc->distance;
MemoryContextSwitchTo(oldCtx);
scan->xs_heaptid = *heaptid;

View File

@@ -5,7 +5,6 @@
#include "access/generic_xlog.h"
#include "catalog/pg_type.h"
#include "catalog/pg_type_d.h"
#include "common/hashfn.h"
#include "fmgr.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
@@ -15,6 +14,12 @@
#include "utils/memdebug.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 130000
#include "common/hashfn.h"
#else
#include "utils/hashutils.h"
#endif
#if PG_VERSION_NUM < 170000
static inline uint64
murmurhash64(uint64 data)
@@ -102,9 +107,10 @@ hash_offset(Size offset)
typedef union
{
HnswElement element;
ItemPointerData indextid;
} HnswUnvisited;
pointerhash_hash *pointers;
offsethash_hash *offsets;
tidhash_hash *tids;
} visited_hash;
/*
* Get the max number of connections in an upper layer for each element in the index
@@ -146,13 +152,37 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
return index_getprocinfo(index, 1, procnum);
}
/*
* Get type
*/
HnswType
HnswGetType(Relation index)
{
FmgrInfo *procinfo = HnswOptionalProcInfo(index, HNSW_TYPE_SUPPORT_PROC);
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
HnswType result;
if (procinfo == NULL)
return HNSW_TYPE_VECTOR;
result = (HnswType) DatumGetInt32(FunctionCall1(procinfo, ObjectIdGetDatum(typid)));
if (result == HNSW_TYPE_UNSUPPORTED)
elog(ERROR, "type not supported for hnsw index");
return result;
}
/*
* Normalize value
*/
Datum
HnswNormValue(const HnswTypeInfo * typeInfo, Oid collation, Datum value)
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum value)
{
return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
if (procinfo == NULL)
return DirectFunctionCall1(l2_normalize, value);
return FunctionCall1Coll(procinfo, collation, value);
}
/*
@@ -164,6 +194,21 @@ HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
}
/*
* Check if a value can be indexed
*/
void
HnswCheckValue(Datum value, HnswType type)
{
if (type == HNSW_TYPE_SPARSEVEC)
{
SparseVector *vec = DatumGetSparseVector(value);
if (vec->nnz > HNSW_MAX_NNZ)
elog(ERROR, "sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ);
}
}
/*
* New buffer
*/
@@ -246,8 +291,6 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
element->level = level;
element->deleted = 0;
/* Start at one to make it easier to find issues */
element->version = 1;
HnswInitNeighbors(base, element, m, allocator);
@@ -296,9 +339,6 @@ HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint)
page = BufferGetPage(buf);
metap = HnswPageGetMeta(page);
if (unlikely(metap->magicNumber != HNSW_MAGIC_NUMBER))
elog(ERROR, "hnsw index is not valid");
if (m != NULL)
*m = metap->m;
@@ -400,7 +440,6 @@ HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element)
etup->type = HNSW_ELEMENT_TUPLE_TYPE;
etup->level = element->level;
etup->deleted = 0;
etup->version = element->version;
for (int i = 0; i < HNSW_HEAPTIDS; i++)
{
if (i < element->heaptidsLength)
@@ -443,7 +482,6 @@ HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m)
}
ntup->count = idx;
ntup->version = e->version;
}
/*
@@ -517,7 +555,6 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
{
element->level = etup->level;
element->deleted = etup->deleted;
element->version = etup->version;
element->neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
element->neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
element->heaptidsLength = 0;
@@ -546,22 +583,25 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
/*
* Load an element and optionally get its distance from q
*/
static void
HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance, HnswElement * element)
void
HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
{
Buffer buf;
Page page;
HnswElementTuple etup;
/* Read vector */
buf = ReadBuffer(index, blkno);
buf = ReadBuffer(index, element->blkno);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, element->offno));
Assert(HnswIsElementTuple(etup));
/* Load element */
HnswLoadElementFromTuple(element, etup, true, loadVec);
/* Calculate distance */
if (distance != NULL)
{
@@ -571,34 +611,17 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, float *distance, Datu
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
}
/* Load element */
if (distance == NULL || maxDistance == NULL || *distance < *maxDistance)
{
if (*element == NULL)
*element = HnswInitElementFromBlock(blkno, offno);
HnswLoadElementFromTuple(*element, etup, true, loadVec);
}
UnlockReleaseBuffer(buf);
}
/*
* Load an element and optionally get its distance from q
*/
void
HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec, float *maxDistance)
{
HnswLoadElementImpl(element->blkno, element->offno, distance, q, index, procinfo, collation, loadVec, maxDistance, &element);
}
/*
* Get the distance for an element
* Get the distance for a candidate
*/
static float
GetElementDistance(char *base, HnswElement element, Datum q, FmgrInfo *procinfo, Oid collation)
GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation)
{
Datum value = HnswGetValue(base, element);
HnswElement hce = HnswPtrAccess(base, hc->element);
Datum value = HnswGetValue(base, hce);
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, value));
}
@@ -606,16 +629,16 @@ GetElementDistance(char *base, HnswElement element, Datum q, FmgrInfo *procinfo,
/*
* Create a candidate for the entry point
*/
HnswSearchCandidate *
HnswCandidate *
HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
{
HnswSearchCandidate *hc = palloc(sizeof(HnswSearchCandidate));
HnswCandidate *hc = palloc(sizeof(HnswCandidate));
HnswPtrStore(base, hc->element, entryPoint);
if (index == NULL)
hc->distance = GetElementDistance(base, entryPoint, q, procinfo, collation);
hc->distance = GetCandidateDistance(base, hc, q, procinfo, collation);
else
HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec, NULL);
HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec);
return hc;
}
@@ -625,25 +648,10 @@ HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index,
static int
CompareNearestCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{
if (HnswGetSearchCandidateConst(c_node, a)->distance < HnswGetSearchCandidateConst(c_node, b)->distance)
if (((const HnswPairingHeapNode *) a)->inner->distance < ((const HnswPairingHeapNode *) b)->inner->distance)
return 1;
if (HnswGetSearchCandidateConst(c_node, a)->distance > HnswGetSearchCandidateConst(c_node, b)->distance)
return -1;
return 0;
}
/*
* Compare discarded candidate distances
*/
static int
CompareNearestDiscardedCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{
if (HnswGetSearchCandidateConst(w_node, a)->distance < HnswGetSearchCandidateConst(w_node, b)->distance)
return 1;
if (HnswGetSearchCandidateConst(w_node, a)->distance > HnswGetSearchCandidateConst(w_node, b)->distance)
if (((const HnswPairingHeapNode *) a)->inner->distance > ((const HnswPairingHeapNode *) b)->inner->distance)
return -1;
return 0;
@@ -655,15 +663,27 @@ CompareNearestDiscardedCandidates(const pairingheap_node *a, const pairingheap_n
static int
CompareFurthestCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{
if (HnswGetSearchCandidateConst(w_node, a)->distance < HnswGetSearchCandidateConst(w_node, b)->distance)
if (((const HnswPairingHeapNode *) a)->inner->distance < ((const HnswPairingHeapNode *) b)->inner->distance)
return -1;
if (HnswGetSearchCandidateConst(w_node, a)->distance > HnswGetSearchCandidateConst(w_node, b)->distance)
if (((const HnswPairingHeapNode *) a)->inner->distance > ((const HnswPairingHeapNode *) b)->inner->distance)
return 1;
return 0;
}
/*
* Create a pairing heap node for a candidate
*/
static HnswPairingHeapNode *
CreatePairingHeapNode(HnswCandidate * c)
{
HnswPairingHeapNode *node = palloc(sizeof(HnswPairingHeapNode));
node->inner = c;
return node;
}
/*
* Init visited
*/
@@ -682,11 +702,11 @@ InitVisited(char *base, visited_hash * v, Relation index, int ef, int m)
* Add to visited
*/
static inline void
AddToVisited(char *base, visited_hash * v, HnswElementPtr elementPtr, Relation index, bool *found)
AddToVisited(char *base, visited_hash * v, HnswCandidate * hc, Relation index, bool *found)
{
if (index != NULL)
{
HnswElement element = HnswPtrAccess(base, elementPtr);
HnswElement element = HnswPtrAccess(base, hc->element);
ItemPointerData indextid;
ItemPointerSet(&indextid, element->blkno, element->offno);
@@ -694,15 +714,23 @@ AddToVisited(char *base, visited_hash * v, HnswElementPtr elementPtr, Relation i
}
else if (base != NULL)
{
HnswElement element = HnswPtrAccess(base, elementPtr);
#if PG_VERSION_NUM >= 130000
HnswElement element = HnswPtrAccess(base, hc->element);
offsethash_insert_hash(v->offsets, HnswPtrOffset(elementPtr), element->hash, found);
offsethash_insert_hash(v->offsets, HnswPtrOffset(hc->element), element->hash, found);
#else
offsethash_insert(v->offsets, HnswPtrOffset(hc->element), found);
#endif
}
else
{
HnswElement element = HnswPtrAccess(base, elementPtr);
#if PG_VERSION_NUM >= 130000
HnswElement element = HnswPtrAccess(base, hc->element);
pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(elementPtr), element->hash, found);
pointerhash_insert_hash(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), element->hash, found);
#else
pointerhash_insert(v->pointers, (uintptr_t) HnswPtrPointer(hc->element), found);
#endif
}
}
@@ -710,168 +738,69 @@ AddToVisited(char *base, visited_hash * v, HnswElementPtr elementPtr, Relation i
* Count element towards ef
*/
static inline bool
CountElement(HnswElement skipElement, HnswElement e)
CountElement(char *base, HnswElement skipElement, HnswCandidate * hc)
{
HnswElement e;
if (skipElement == NULL)
return true;
/* Ensure does not access heaptidsLength during in-memory build */
pg_memory_barrier();
/* Keep scan-build happy on Mac x86-64 */
Assert(e);
e = HnswPtrAccess(base, hc->element);
return e->heaptidsLength != 0;
}
/*
* Load unvisited neighbors from memory
*/
static void
HnswLoadUnvisitedFromMemory(char *base, HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, int lc, HnswNeighborArray * localNeighborhood, Size neighborhoodSize)
{
/* Get the neighborhood at layer lc */
HnswNeighborArray *neighborhood = HnswGetNeighbors(base, element, lc);
/* Copy neighborhood to local memory */
LWLockAcquire(&element->lock, LW_SHARED);
memcpy(localNeighborhood, neighborhood, neighborhoodSize);
LWLockRelease(&element->lock);
*unvisitedLength = 0;
for (int i = 0; i < localNeighborhood->length; i++)
{
HnswCandidate *hc = &localNeighborhood->items[i];
bool found;
AddToVisited(base, v, hc->element, NULL, &found);
if (!found)
unvisited[(*unvisitedLength)++].element = HnswPtrAccess(base, hc->element);
}
}
/*
* Load unvisited neighbors from disk
*/
static void
HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *unvisitedLength, visited_hash * v, Relation index, int m, int lm, int lc)
{
Buffer buf;
Page page;
HnswNeighborTuple ntup;
int start;
ItemPointerData indextids[HNSW_MAX_M * 2];
*unvisitedLength = 0;
buf = ReadBuffer(index, element->neighborPage);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
/*
* Ensure the neighbor tuple has not been deleted or replaced between
* index scan iterations
*/
if (ntup->version != element->version || ntup->count != (element->level + 2) * m)
{
UnlockReleaseBuffer(buf);
return;
}
/* Copy to minimize lock time */
start = (element->level - lc) * m;
memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
UnlockReleaseBuffer(buf);
for (int i = 0; i < lm; i++)
{
ItemPointer indextid = &indextids[i];
bool found;
if (!ItemPointerIsValid(indextid))
break;
tidhash_insert(v->tids, *indextid, &found);
if (!found)
unvisited[(*unvisitedLength)++].indextid = *indextid;
}
}
/*
* Algorithm 2 from paper
*/
List *
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples)
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
{
List *w = NIL;
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
pairingheap *W = pairingheap_allocate(CompareFurthestCandidates, NULL);
int wlen = 0;
visited_hash vh;
visited_hash v;
ListCell *lc2;
HnswNeighborArray *localNeighborhood = NULL;
Size neighborhoodSize = 0;
int lm = HnswGetLayerM(m, lc);
HnswUnvisited *unvisited = palloc(lm * sizeof(HnswUnvisited));
int unvisitedLength;
HnswNeighborArray *neighborhoodData = NULL;
Size neighborhoodSize;
if (v == NULL)
{
v = &vh;
initVisited = true;
}
if (initVisited)
{
InitVisited(base, v, index, ef, m);
if (discarded != NULL)
*discarded = pairingheap_allocate(CompareNearestDiscardedCandidates, NULL);
}
InitVisited(base, &v, index, ef, m);
/* Create local memory for neighborhood if needed */
if (index == NULL)
{
neighborhoodSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm);
localNeighborhood = palloc(neighborhoodSize);
neighborhoodSize = HNSW_NEIGHBOR_ARRAY_SIZE(HnswGetLayerM(m, lc));
neighborhoodData = palloc(neighborhoodSize);
}
/* Add entry points to v, C, and W */
foreach(lc2, ep)
{
HnswSearchCandidate *hc = (HnswSearchCandidate *) lfirst(lc2);
HnswCandidate *hc = (HnswCandidate *) lfirst(lc2);
bool found;
if (initVisited)
{
AddToVisited(base, v, hc->element, index, &found);
AddToVisited(base, &v, hc, index, &found);
if (tuples != NULL)
(*tuples)++;
}
pairingheap_add(C, &hc->c_node);
pairingheap_add(W, &hc->w_node);
pairingheap_add(C, &(CreatePairingHeapNode(hc)->ph_node));
pairingheap_add(W, &(CreatePairingHeapNode(hc)->ph_node));
/*
* Do not count elements being deleted towards ef when vacuuming. It
* would be ideal to do this for inserts as well, but this could
* affect insert performance.
*/
if (CountElement(skipElement, HnswPtrAccess(base, hc->element)))
if (CountElement(base, skipElement, hc))
wlen++;
}
while (!pairingheap_is_empty(C))
{
HnswSearchCandidate *c = HnswGetSearchCandidate(c_node, pairingheap_remove_first(C));
HnswSearchCandidate *f = HnswGetSearchCandidate(w_node, pairingheap_first(W));
HnswNeighborArray *neighborhood;
HnswCandidate *c = ((HnswPairingHeapNode *) pairingheap_remove_first(C))->inner;
HnswCandidate *f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
HnswElement cElement;
if (c->distance > f->distance)
@@ -879,80 +808,70 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
cElement = HnswPtrAccess(base, c->element);
if (HnswPtrIsNull(base, cElement->neighbors))
HnswLoadNeighbors(cElement, index, m);
/* Get the neighborhood at layer lc */
neighborhood = HnswGetNeighbors(base, cElement, lc);
/* Copy neighborhood to local memory if needed */
if (index == NULL)
HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize);
else
HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
if (tuples != NULL)
(*tuples) += unvisitedLength;
for (int i = 0; i < unvisitedLength; i++)
{
HnswElement eElement;
HnswSearchCandidate *e;
float eDistance;
bool alwaysAdd = wlen < ef;
LWLockAcquire(&cElement->lock, LW_SHARED);
memcpy(neighborhoodData, neighborhood, neighborhoodSize);
LWLockRelease(&cElement->lock);
neighborhood = neighborhoodData;
}
f = HnswGetSearchCandidate(w_node, pairingheap_first(W));
for (int i = 0; i < neighborhood->length; i++)
{
HnswCandidate *e = &neighborhood->items[i];
bool visited;
if (index == NULL)
AddToVisited(base, &v, e, index, &visited);
if (!visited)
{
eElement = unvisited[i].element;
eDistance = GetElementDistance(base, eElement, q, procinfo, collation);
}
else
{
ItemPointer indextid = &unvisited[i].indextid;
BlockNumber blkno = ItemPointerGetBlockNumber(indextid);
OffsetNumber offno = ItemPointerGetOffsetNumber(indextid);
float eDistance;
HnswElement eElement = HnswPtrAccess(base, e->element);
/* Avoid any allocations if not adding */
eElement = NULL;
HnswLoadElementImpl(blkno, offno, &eDistance, &q, index, procinfo, collation, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
}
f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
{
if (discarded != NULL)
if (index == NULL)
eDistance = GetCandidateDistance(base, e, q, procinfo, collation);
else
HnswLoadElement(eElement, &eDistance, &q, index, procinfo, collation, inserting);
Assert(!eElement->deleted);
/* Make robust to issues */
if (eElement->level < lc)
continue;
if (eDistance < f->distance || wlen < ef)
{
/* Create a new candidate */
e = palloc(sizeof(HnswSearchCandidate));
HnswPtrStore(base, e->element, eElement);
e->distance = eDistance;
pairingheap_add(*discarded, &e->w_node);
}
/* Copy e */
HnswCandidate *ec = palloc(sizeof(HnswCandidate));
continue;
}
HnswPtrStore(base, ec->element, eElement);
ec->distance = eDistance;
/* Make robust to issues */
if (eElement->level < lc)
continue;
pairingheap_add(C, &(CreatePairingHeapNode(ec)->ph_node));
pairingheap_add(W, &(CreatePairingHeapNode(ec)->ph_node));
/* Create a new candidate */
e = palloc(sizeof(HnswSearchCandidate));
HnswPtrStore(base, e->element, eElement);
e->distance = eDistance;
pairingheap_add(C, &e->c_node);
pairingheap_add(W, &e->w_node);
/*
* Do not count elements being deleted towards ef when
* vacuuming. It would be ideal to do this for inserts as
* well, but this could affect insert performance.
*/
if (CountElement(base, skipElement, e))
{
wlen++;
/*
* Do not count elements being deleted towards ef when vacuuming.
* It would be ideal to do this for inserts as well, but this
* could affect insert performance.
*/
if (CountElement(skipElement, eElement))
{
wlen++;
/* No need to decrement wlen */
if (wlen > ef)
{
HnswSearchCandidate *d = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
if (discarded != NULL)
pairingheap_add(*discarded, &d->w_node);
/* No need to decrement wlen */
if (wlen > ef)
pairingheap_remove_first(W);
}
}
}
}
@@ -961,7 +880,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
/* Add each element of W to w */
while (!pairingheap_is_empty(W))
{
HnswSearchCandidate *hc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
HnswCandidate *hc = ((HnswPairingHeapNode *) pairingheap_remove_first(W))->inner;
w = lappend(w, hc);
}
@@ -973,10 +892,17 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
* Compare candidate distances with pointer tie-breaker
*/
static int
#if PG_VERSION_NUM >= 130000
CompareCandidateDistances(const ListCell *a, const ListCell *b)
{
HnswCandidate *hca = lfirst(a);
HnswCandidate *hcb = lfirst(b);
#else
CompareCandidateDistances(const void *a, const void *b)
{
HnswCandidate *hca = lfirst(*(ListCell **) a);
HnswCandidate *hcb = lfirst(*(ListCell **) b);
#endif
if (hca->distance < hcb->distance)
return 1;
@@ -997,10 +923,17 @@ CompareCandidateDistances(const ListCell *a, const ListCell *b)
* Compare candidate distances with offset tie-breaker
*/
static int
#if PG_VERSION_NUM >= 130000
CompareCandidateDistancesOffset(const ListCell *a, const ListCell *b)
{
HnswCandidate *hca = lfirst(a);
HnswCandidate *hcb = lfirst(b);
#else
CompareCandidateDistancesOffset(const void *a, const void *b)
{
HnswCandidate *hca = lfirst(*(ListCell **) a);
HnswCandidate *hcb = lfirst(*(ListCell **) b);
#endif
if (hca->distance < hcb->distance)
return 1;
@@ -1208,9 +1141,9 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
HnswElement hc3Element = HnswPtrAccess(base, hc3->element);
if (HnswPtrIsNull(base, hc3Element->value))
HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true, NULL);
HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true);
else
hc3->distance = GetElementDistance(base, hc3Element, q, procinfo, collation);
hc3->distance = GetCandidateDistance(base, hc3, q, procinfo, collation);
/* Prune element if being deleted */
if (hc3Element->heaptidsLength == 0)
@@ -1282,6 +1215,7 @@ RemoveElements(char *base, List *w, HnswElement skipElement)
return w2;
}
#if PG_VERSION_NUM >= 130000
/*
* Precompute hash
*/
@@ -1297,6 +1231,7 @@ PrecomputeHash(char *base, HnswElement element)
else
element->hash = hash_offset(HnswPtrOffset(ptr));
}
#endif
/*
* Algorithm 1 from paper
@@ -1311,9 +1246,11 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
Datum q = HnswGetValue(base, element);
HnswElement skipElement = existing ? element : NULL;
#if PG_VERSION_NUM >= 130000
/* Precompute hash */
if (index == NULL)
PrecomputeHash(base, element);
#endif
/* No neighbors if no entry point */
if (entryPoint == NULL)
@@ -1326,7 +1263,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
/* 1st phase: greedy search to insert level */
for (int lc = entryLevel; lc >= level + 1; lc--)
{
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement, NULL, NULL, true, NULL);
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
ep = w;
}
@@ -1342,27 +1279,16 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
{
int lm = HnswGetLayerM(m, lc);
List *neighbors;
List *lw = NIL;
ListCell *lc2;
List *lw;
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement, NULL, NULL, true, NULL);
/* Convert search candidates to candidates */
foreach(lc2, w)
{
HnswSearchCandidate *sc = lfirst(lc2);
HnswCandidate *hc = palloc(sizeof(HnswCandidate));
hc->element = sc->element;
hc->distance = sc->distance;
lw = lappend(lw, hc);
}
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
/* Elements being deleted or skipped can help with search */
/* but should be removed before selecting neighbors */
if (index != NULL)
lw = RemoveElements(base, lw, skipElement);
lw = RemoveElements(base, w, skipElement);
else
lw = w;
/*
* Candidates are sorted, but not deterministically. Could set
@@ -1377,78 +1303,28 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
}
}
PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
static void
SparsevecCheckValue(Pointer v)
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_hnsw_support);
Datum
halfvec_hnsw_support(PG_FUNCTION_ARGS)
{
SparseVector *vec = (SparseVector *) v;
PG_RETURN_INT32(HNSW_TYPE_HALFVEC);
};
if (vec->nnz > HNSW_MAX_NNZ)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ)));
}
/*
* Get type info
*/
const HnswTypeInfo *
HnswGetTypeInfo(Relation index)
PGDLLEXPORT PG_FUNCTION_INFO_V1(bit_hnsw_support);
Datum
bit_hnsw_support(PG_FUNCTION_ARGS)
{
FmgrInfo *procinfo = HnswOptionalProcInfo(index, HNSW_TYPE_INFO_PROC);
Oid typid = PG_GETARG_OID(0);
if (procinfo == NULL)
{
static const HnswTypeInfo typeInfo = {
.maxDimensions = HNSW_MAX_DIM,
.normalize = l2_normalize,
.checkValue = NULL
};
return (&typeInfo);
}
if (typid == BITOID)
PG_RETURN_INT32(HNSW_TYPE_BIT);
else
return (const HnswTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid));
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_halfvec_support);
Datum
hnsw_halfvec_support(PG_FUNCTION_ARGS)
{
static const HnswTypeInfo typeInfo = {
.maxDimensions = HNSW_MAX_DIM * 2,
.normalize = halfvec_l2_normalize,
.checkValue = NULL
};
PG_RETURN_POINTER(&typeInfo);
PG_RETURN_INT32(HNSW_TYPE_UNSUPPORTED);
};
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_hnsw_support);
Datum
hnsw_bit_support(PG_FUNCTION_ARGS)
sparsevec_hnsw_support(PG_FUNCTION_ARGS)
{
static const HnswTypeInfo typeInfo = {
.maxDimensions = HNSW_MAX_DIM * 32,
.normalize = NULL,
.checkValue = NULL
};
PG_RETURN_POINTER(&typeInfo);
};
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_sparsevec_support);
Datum
hnsw_sparsevec_support(PG_FUNCTION_ARGS)
{
static const HnswTypeInfo typeInfo = {
.maxDimensions = SPARSEVEC_MAX_DIM,
.normalize = sparsevec_l2_normalize,
.checkValue = SparsevecCheckValue
};
PG_RETURN_POINTER(&typeInfo);
PG_RETURN_INT32(HNSW_TYPE_SPARSEVEC);
};

View File

@@ -256,7 +256,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL);
HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
@@ -294,7 +294,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
* is outdated, this can remove connections at higher levels in
* the graph until they are repaired, but this should be fine.
*/
HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL);
HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
if (NeedsUpdated(vacuumstate, entryPoint))
{
@@ -527,14 +527,6 @@ MarkDeleted(HnswVacuumState * vacuumstate)
for (int i = 0; i < ntup->count; i++)
ItemPointerSetInvalid(&ntup->indextids[i]);
/* Increment version */
/* This is used to avoid incorrect reads for iterative scans */
/* Reserve some bits for future use */
etup->version++;
if (etup->version > 15)
etup->version = 1;
ntup->version = etup->version;
/*
* We modified the tuples in place, no need to call
* PageIndexTupleOverwrite

View File

@@ -26,6 +26,12 @@
#include "pgstat.h"
#endif
#if PG_VERSION_NUM >= 130000
#define CALLBACK_ITEM_POINTER ItemPointer tid
#else
#define CALLBACK_ITEM_POINTER HeapTuple hup
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_status.h"
#include "utils/wait_event.h"
@@ -57,7 +63,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
return;
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
value = IvfflatNormValue(buildstate->normalizeprocinfo, buildstate->collation, value);
}
if (samples->length < targsamples)
@@ -90,7 +96,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
* Callback for sampling
*/
static void
SampleCallback(Relation index, ItemPointer tid, Datum *values,
SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state)
{
IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
@@ -155,7 +161,7 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
if (!IvfflatCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
return;
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
value = IvfflatNormValue(buildstate->normalizeprocinfo, buildstate->collation, value);
}
/* Find the list that minimizes the distance */
@@ -201,12 +207,16 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
* Callback for table_index_build_scan
*/
static void
BuildCallback(Relation index, ItemPointer tid, Datum *values,
BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state)
{
IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
MemoryContext oldCtx;
#if PG_VERSION_NUM < 130000
ItemPointer tid = &hup->t_self;
#endif
/* Skip nulls */
if (isnull[0])
return;
@@ -309,36 +319,62 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
}
}
/*
* Get max dimensions
*/
static int
GetMaxDimensions(IvfflatType type)
{
int maxDimensions = IVFFLAT_MAX_DIM;
if (type == IVFFLAT_TYPE_HALFVEC)
maxDimensions *= 2;
else if (type == IVFFLAT_TYPE_BIT)
maxDimensions *= 32;
return maxDimensions;
}
/*
* Get item size
*/
static Size
GetItemSize(IvfflatType type, int dimensions)
{
if (type == IVFFLAT_TYPE_VECTOR)
return VECTOR_SIZE(dimensions);
else if (type == IVFFLAT_TYPE_HALFVEC)
return HALFVEC_SIZE(dimensions);
else if (type == IVFFLAT_TYPE_BIT)
return VARBITTOTALLEN(dimensions);
else
elog(ERROR, "Unsupported type");
}
/*
* Initialize the build state
*/
static void
InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo)
{
int maxDimensions;
buildstate->heap = heap;
buildstate->index = index;
buildstate->indexInfo = indexInfo;
buildstate->typeInfo = IvfflatGetTypeInfo(index);
buildstate->type = IvfflatGetType(index);
buildstate->lists = IvfflatGetLists(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
/* Disallow varbit since require fixed dimensions */
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("type not supported for ivfflat index")));
maxDimensions = GetMaxDimensions(buildstate->type);
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("column does not have dimensions")));
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("column cannot have more than %d dimensions for ivfflat index", buildstate->typeInfo->maxDimensions)));
if (buildstate->dimensions > maxDimensions)
elog(ERROR, "column cannot have more than %d dimensions for ivfflat index", maxDimensions);
buildstate->reltuples = 0;
buildstate->indtuples = 0;
@@ -347,13 +383,12 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
buildstate->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
buildstate->kmeansnormprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
buildstate->normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
buildstate->collation = index->rd_indcollation[0];
/* Require more than one dimension for spherical k-means */
if (buildstate->kmeansnormprocinfo != NULL && buildstate->dimensions == 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions must be greater than one for this opclass")));
elog(ERROR, "dimensions must be greater than one for this opclass");
/* Create tuple description for sorting */
buildstate->tupdesc = CreateTemplateTupleDesc(3);
@@ -363,7 +398,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsVirtual);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->typeInfo->itemSize(buildstate->dimensions));
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, GetItemSize(buildstate->type, buildstate->dimensions));
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -433,7 +468,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
}
/* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo));
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->type));
/* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples);
@@ -488,13 +523,10 @@ CreateListPages(Relation index, VectorArray centers, int dimensions,
{
OffsetNumber offno;
/* Zero memory for each list */
MemSet(list, 0, listSize);
/* Load list */
list->startPage = InvalidBlockNumber;
list->insertPage = InvalidBlockNumber;
memcpy(&list->center, VectorArrayGet(centers, i), VARSIZE_ANY(VectorArrayGet(centers, i)));
memcpy(&list->center, VectorArrayGet(centers, i), centers->itemsize);
/* Ensure free space */
if (PageGetFreeSpace(page) < listSize)
@@ -560,20 +592,6 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
}
#endif
/*
* Initialize build sort state
*/
static Tuplesortstate *
InitBuildSortState(TupleDesc tupdesc, int memory, SortCoordinate coordinate)
{
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, memory, coordinate, false);
}
/*
* Within leader, wait for end of heap scan
*/
@@ -621,6 +639,12 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
double reltuples;
IndexInfo *indexInfo;
/* Sort options, which must match AssignTuples */
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
/* Initialize local tuplesort coordination state */
coordinate = palloc0(sizeof(SortCoordinateData));
coordinate->isWorker = true;
@@ -633,7 +657,7 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
InitBuildState(&buildstate, ivfspool->heap, ivfspool->index, indexInfo);
memcpy(buildstate.centers->items, ivfcenters, buildstate.centers->itemsize * buildstate.centers->maxlen);
buildstate.centers->length = buildstate.centers->maxlen;
ivfspool->sortstate = InitBuildSortState(buildstate.tupdesc, sortmem, coordinate);
ivfspool->sortstate = tuplesort_begin_heap(buildstate.tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, sortmem, coordinate, false);
buildstate.sortstate = ivfspool->sortstate;
scan = table_beginscan_parallel(ivfspool->heap,
ParallelTableScanFromIvfflatShared(ivfshared));
@@ -930,6 +954,12 @@ AssignTuples(IvfflatBuildState * buildstate)
int parallel_workers = 0;
SortCoordinate coordinate = NULL;
/* Sort options, which must match IvfflatParallelScanAndSort */
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN);
/* Calculate parallel workers */
@@ -950,7 +980,7 @@ AssignTuples(IvfflatBuildState * buildstate)
}
/* Begin serial/leader tuplesort */
buildstate->sortstate = InitBuildSortState(buildstate->tupdesc, maintenance_work_mem, coordinate);
buildstate->sortstate = tuplesort_begin_heap(buildstate->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, maintenance_work_mem, coordinate, false);
/* Add tuples to sort */
if (buildstate->heap != NULL)
@@ -1006,10 +1036,6 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum);
/* Write WAL for initialization fork since GenericXLog functions do not */
if (forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
FreeBuildState(buildstate);
}

View File

@@ -7,7 +7,6 @@
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "ivfflat.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
@@ -27,7 +26,11 @@ IvfflatInit(void)
{
ivfflat_relopt_kind = add_reloption_kind();
add_int_reloption(ivfflat_relopt_kind, "lists", "Number of inverted lists",
IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, AccessExclusiveLock);
IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
"Valid range is 1..lists.", &ivfflat_probes,
@@ -75,8 +78,8 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
/* Never use index without order */
if (path->indexorderbys == NULL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
*indexStartupCost = DBL_MAX;
*indexTotalCost = DBL_MAX;
*indexSelectivity = 0;
*indexCorrelation = 0;
*indexPages = 0;
@@ -120,6 +123,13 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
}
/*
* If the list selectivity is lower than what is returned from the generic
* cost estimator, use that.
*/
if (ratio < costs.indexSelectivity)
costs.indexSelectivity = ratio;
/* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost;
*indexTotalCost = costs.indexTotalCost;
@@ -138,10 +148,23 @@ ivfflatoptions(Datum reloptions, bool validate)
{"lists", RELOPT_TYPE_INT, offsetof(IvfflatOptions, lists)},
};
#if PG_VERSION_NUM >= 130000
return (bytea *) build_reloptions(reloptions, validate,
ivfflat_relopt_kind,
sizeof(IvfflatOptions),
tab, lengthof(tab));
#else
relopt_value *options;
int numoptions;
IvfflatOptions *rdopts;
options = parseRelOptions(reloptions, validate, ivfflat_relopt_kind, &numoptions);
rdopts = allocateReloptStruct(sizeof(IvfflatOptions), options, numoptions);
fillRelOptions((void *) rdopts, sizeof(IvfflatOptions), options, numoptions,
validate, tab, lengthof(tab));
return (bytea *) rdopts;
#endif
}
/*
@@ -158,15 +181,17 @@ ivfflatvalidate(Oid opclassoid)
*
* See https://www.postgresql.org/docs/current/index-api.html
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
Datum
ivfflathandler(PG_FUNCTION_ARGS)
{
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
amroutine->amsupport = 5;
amroutine->amsupport = 6;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0;
#endif
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
amroutine->amcanbackward = false; /* can change direction mid-scan */
@@ -179,24 +204,17 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amclusterable = false;
amroutine->ampredlocks = false;
amroutine->amcanparallel = false;
#if PG_VERSION_NUM >= 170000
amroutine->amcanbuildparallel = true;
#endif
amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
#if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false;
#endif
amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
#endif
amroutine->amkeytype = InvalidOid;
/* Interface functions */
amroutine->ambuild = ivfflatbuild;
amroutine->ambuildempty = ivfflatbuildempty;
amroutine->aminsert = ivfflatinsert;
#if PG_VERSION_NUM >= 170000
amroutine->aminsertcleanup = NULL;
#endif
amroutine->ambulkdelete = ivfflatbulkdelete;
amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */

View File

@@ -28,7 +28,8 @@
#define IVFFLAT_NORM_PROC 2
#define IVFFLAT_KMEANS_DISTANCE_PROC 3
#define IVFFLAT_KMEANS_NORM_PROC 4
#define IVFFLAT_TYPE_INFO_PROC 5
#define IVFFLAT_NORMALIZE_PROC 5
#define IVFFLAT_TYPE_SUPPORT_PROC 6
#define IVFFLAT_VERSION 1
#define IVFFLAT_MAGIC_NUMBER 0x14FF1A7
@@ -44,6 +45,14 @@
#define IVFFLAT_MAX_LISTS 32768
#define IVFFLAT_DEFAULT_PROBES 1
typedef enum IvfflatType
{
IVFFLAT_TYPE_VECTOR,
IVFFLAT_TYPE_HALFVEC,
IVFFLAT_TYPE_BIT,
IVFFLAT_TYPE_UNSUPPORTED
} IvfflatType;
/* Build phases */
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
#define PROGRESS_IVFFLAT_PHASE_KMEANS 2
@@ -149,22 +158,13 @@ typedef struct IvfflatLeader
char *ivfcenters;
} IvfflatLeader;
typedef struct IvfflatTypeInfo
{
int maxDimensions;
Datum (*normalize) (PG_FUNCTION_ARGS);
Size (*itemSize) (int dimensions);
void (*updateCenter) (Pointer v, int dimensions, float *x);
void (*sumCenter) (Pointer v, float *x);
} IvfflatTypeInfo;
typedef struct IvfflatBuildState
{
/* Info */
Relation heap;
Relation index;
IndexInfo *indexInfo;
const IvfflatTypeInfo *typeInfo;
IvfflatType type;
/* Settings */
int dimensions;
@@ -178,6 +178,7 @@ typedef struct IvfflatBuildState
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *kmeansnormprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
/* Variables */
@@ -245,7 +246,6 @@ typedef struct IvfflatScanList
typedef struct IvfflatScanOpaqueData
{
const IvfflatTypeInfo *typeInfo;
int probes;
int dimensions;
bool first;
@@ -253,13 +253,13 @@ typedef struct IvfflatScanOpaqueData
/* Sorting */
Tuplesortstate *sortstate;
TupleDesc tupdesc;
TupleTableSlot *vslot;
TupleTableSlot *mslot;
BufferAccessStrategy bas;
TupleTableSlot *slot;
bool isnull;
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
Datum (*distfunc) (FmgrInfo *flinfo, Oid collation, Datum arg1, Datum arg2);
@@ -271,27 +271,17 @@ typedef struct IvfflatScanOpaqueData
typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
#define VECTOR_ARRAY_SIZE(_length, _size) (sizeof(VectorArrayData) + (_length) * MAXALIGN(_size))
/* Use functions instead of macros to avoid double evaluation */
static inline Pointer
VectorArrayGet(VectorArray arr, int offset)
{
return ((char *) arr->items) + (offset * arr->itemsize);
}
static inline void
VectorArraySet(VectorArray arr, int offset, Pointer val)
{
memcpy(VectorArrayGet(arr, offset), val, VARSIZE_ANY(val));
}
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) (_arr)->items + (_offset) * (_arr)->itemsize)
#define VectorArrayGet(_arr, _offset) VECTOR_ARRAY_OFFSET(_arr, _offset)
#define VectorArraySet(_arr, _offset, _val) memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, (_arr)->itemsize)
/* Methods */
VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
void VectorArrayFree(VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, IvfflatType type);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
IvfflatType IvfflatGetType(Relation index);
Datum IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum value);
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
@@ -302,7 +292,6 @@ Buffer IvfflatNewBuffer(Relation index, ForkNumber forkNum);
void IvfflatInitPage(Buffer buf, Page page);
void IvfflatInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state);
void IvfflatInit(void);
const IvfflatTypeInfo *IvfflatGetTypeInfo(Relation index);
PGDLLEXPORT void IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc);
/* Index access methods */

View File

@@ -67,7 +67,6 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
static void
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
{
const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
IndexTuple itup;
Datum value;
FmgrInfo *normprocinfo;
@@ -91,12 +90,9 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
if (!IvfflatCheckNorm(normprocinfo, collation, value))
return;
value = IvfflatNormValue(typeInfo, collation, value);
value = IvfflatNormValue(IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC), collation, value);
}
/* Ensure index is valid */
IvfflatGetMetaPageInfo(index, NULL, NULL);
/* Find the insert page - sets the page and list info */
FindInsertPage(index, values, &insertPage, &listInfo);
Assert(BlockNumberIsValid(insertPage));

View File

@@ -92,7 +92,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
* Norm centers
*/
static void
NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers)
NormCenters(FmgrInfo *normalizeprocinfo, Oid collation, VectorArray centers)
{
MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat norm temporary context",
@@ -102,7 +102,7 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
for (int j = 0; j < centers->length; j++)
{
Datum center = PointerGetDatum(VectorArrayGet(centers, j));
Datum newCenter = IvfflatNormValue(typeInfo, collation, center);
Datum newCenter = IvfflatNormValue(normalizeprocinfo, collation, center);
Size size = VARSIZE_ANY(DatumGetPointer(newCenter));
if (size > centers->itemsize)
@@ -117,31 +117,120 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
}
/*
* Quick approach if we have no data
* Compare vectors
*/
static int
CompareVectors(const void *a, const void *b)
{
return vector_cmp_internal((Vector *) a, (Vector *) b);
}
/*
* Compare half vectors
*/
static int
CompareHalfVectors(const void *a, const void *b)
{
return halfvec_cmp_internal((HalfVector *) a, (HalfVector *) b);
}
/*
* Compare bit vectors
*/
static int
CompareBitVectors(const void *a, const void *b)
{
return DirectFunctionCall2(bitcmp, VarBitPGetDatum((VarBit *) a), VarBitPGetDatum((VarBit *) b));
}
/*
* Sort vector array
*/
static void
RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
SortVectorArray(VectorArray arr, IvfflatType type)
{
if (type == IVFFLAT_TYPE_VECTOR)
qsort(arr->items, arr->length, arr->itemsize, CompareVectors);
else if (type == IVFFLAT_TYPE_HALFVEC)
qsort(arr->items, arr->length, arr->itemsize, CompareHalfVectors);
else if (type == IVFFLAT_TYPE_BIT)
qsort(arr->items, arr->length, arr->itemsize, CompareBitVectors);
else
elog(ERROR, "Unsupported type");
}
/*
* Quick approach if we have little data
*/
static void
QuickCenters(Relation index, VectorArray samples, VectorArray centers, IvfflatType type)
{
int dimensions = centers->dim;
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
Oid collation = index->rd_indcollation[0];
float *x = (float *) palloc(sizeof(float) * dimensions);
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
FmgrInfo *normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
/* Fill with random data */
/* Copy existing vectors while avoiding duplicates */
if (samples->length > 0)
{
SortVectorArray(samples, type);
for (int i = 0; i < samples->length; i++)
{
Datum vec = PointerGetDatum(VectorArrayGet(samples, i));
if (i == 0 || !datumIsEqual(vec, PointerGetDatum(VectorArrayGet(samples, i - 1)), false, -1))
{
VectorArraySet(centers, centers->length, DatumGetPointer(vec));
centers->length++;
}
}
}
/* Fill remaining with random data */
while (centers->length < centers->maxlen)
{
Pointer center = VectorArrayGet(centers, centers->length);
Datum center = PointerGetDatum(VectorArrayGet(centers, centers->length));
for (int i = 0; i < dimensions; i++)
x[i] = (float) RandomDouble();
if (type == IVFFLAT_TYPE_VECTOR)
{
Vector *vec = DatumGetVector(center);
typeInfo->updateCenter(center, dimensions, x);
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
for (int j = 0; j < dimensions; j++)
vec->x[j] = RandomDouble();
}
else if (type == IVFFLAT_TYPE_HALFVEC)
{
HalfVector *vec = DatumGetHalfVector(center);
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
for (int j = 0; j < dimensions; j++)
vec->x[j] = Float4ToHalfUnchecked((float) RandomDouble());
}
else if (type == IVFFLAT_TYPE_BIT)
{
VarBit *vec = DatumGetVarBitP(center);
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
for (int j = 0; j < dimensions; j++)
VARBITS(vec)[j / dimensions] |= (RandomDouble() > 0.5 ? 1 : 0) << (7 - (j % 8));
}
else
elog(ERROR, "Unsupported type");
centers->length++;
}
/* Fine if existing vectors are normalized twice */
if (normprocinfo != NULL)
NormCenters(typeInfo, collation, centers);
NormCenters(normalizeprocinfo, collation, centers);
}
#ifdef IVFFLAT_MEMORY
@@ -151,8 +240,12 @@ RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeI
static void
ShowMemoryUsage(MemoryContext context, Size estimatedSize)
{
#if PG_VERSION_NUM >= 130000
elog(INFO, "total memory: %zu MB",
MemoryContextMemAllocated(context, true) / (1024 * 1024));
#else
MemoryContextStats(context);
#endif
elog(INFO, "estimated memory: %zu MB", estimatedSize / (1024 * 1024));
}
#endif
@@ -161,27 +254,82 @@ ShowMemoryUsage(MemoryContext context, Size estimatedSize)
* Sum centers
*/
static void
SumCenters(VectorArray samples, float *agg, int *closestCenters, const IvfflatTypeInfo * typeInfo)
SumCenters(VectorArray samples, VectorArray aggCenters, int *closestCenters, IvfflatType type)
{
for (int j = 0; j < samples->length; j++)
{
float *x = agg + ((int64) closestCenters[j] * samples->dim);
int dimensions = aggCenters->dim;
int numSamples = samples->length;
typeInfo->sumCenter(VectorArrayGet(samples, j), x);
if (type == IVFFLAT_TYPE_VECTOR)
{
for (int j = 0; j < numSamples; j++)
{
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, closestCenters[j]);
Vector *vec = (Vector *) VectorArrayGet(samples, j);
for (int k = 0; k < dimensions; k++)
aggCenter->x[k] += vec->x[k];
}
}
else if (type == IVFFLAT_TYPE_HALFVEC)
{
for (int j = 0; j < numSamples; j++)
{
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, closestCenters[j]);
HalfVector *vec = (HalfVector *) VectorArrayGet(samples, j);
for (int k = 0; k < dimensions; k++)
aggCenter->x[k] += HalfToFloat4(vec->x[k]);
}
}
else if (type == IVFFLAT_TYPE_BIT)
{
for (int j = 0; j < numSamples; j++)
{
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, closestCenters[j]);
VarBit *vec = (VarBit *) VectorArrayGet(samples, j);
for (int k = 0; k < dimensions; k++)
aggCenter->x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
}
}
else
elog(ERROR, "Unsupported type");
}
/*
* Update centers
* Set new centers
*/
static void
UpdateCenters(float *agg, VectorArray centers, const IvfflatTypeInfo * typeInfo)
SetNewCenters(VectorArray aggCenters, VectorArray newCenters, IvfflatType type)
{
for (int j = 0; j < centers->length; j++)
{
float *x = agg + ((int64) j * centers->dim);
int dimensions = aggCenters->dim;
int numCenters = aggCenters->maxlen;
typeInfo->updateCenter(VectorArrayGet(centers, j), centers->dim, x);
if (type == IVFFLAT_TYPE_HALFVEC)
{
for (int j = 0; j < numCenters; j++)
{
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, j);
HalfVector *newCenter = (HalfVector *) VectorArrayGet(newCenters, j);
for (int k = 0; k < dimensions; k++)
newCenter->x[k] = Float4ToHalfUnchecked(aggCenter->x[k]);
}
}
else if (type == IVFFLAT_TYPE_BIT)
{
for (int j = 0; j < numCenters; j++)
{
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, j);
VarBit *newCenter = (VarBit *) VectorArrayGet(newCenters, j);
unsigned char *nx = VARBITS(newCenter);
for (uint32 k = 0; k < VARBITBYTES(newCenter); k++)
nx[k] = 0;
for (int k = 0; k < dimensions; k++)
nx[k / 8] |= (aggCenter->x[k] > 0.5) << (7 - (k % 8));
}
}
}
@@ -189,25 +337,25 @@ UpdateCenters(float *agg, VectorArray centers, const IvfflatTypeInfo * typeInfo)
* Compute new centers
*/
static void
ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *centerCounts, int *closestCenters, FmgrInfo *normprocinfo, Oid collation, const IvfflatTypeInfo * typeInfo)
ComputeNewCenters(VectorArray samples, VectorArray aggCenters, VectorArray newCenters, int *centerCounts, int *closestCenters, FmgrInfo *normprocinfo, FmgrInfo *normalizeprocinfo, Oid collation, IvfflatType type)
{
int dimensions = newCenters->dim;
int numCenters = newCenters->length;
int dimensions = aggCenters->dim;
int numCenters = aggCenters->maxlen;
int numSamples = samples->length;
/* Reset sum and count */
for (int j = 0; j < numCenters; j++)
{
float *x = agg + ((int64) j * dimensions);
Vector *vec = (Vector *) VectorArrayGet(aggCenters, j);
for (int k = 0; k < dimensions; k++)
x[k] = 0.0;
vec->x[k] = 0.0;
centerCounts[j] = 0;
}
/* Increment sum of closest center */
SumCenters(samples, agg, closestCenters, typeInfo);
SumCenters(samples, aggCenters, closestCenters, type);
/* Increment count of closest center */
for (int j = 0; j < numSamples; j++)
@@ -216,7 +364,7 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
/* Divide sum by count */
for (int j = 0; j < numCenters; j++)
{
float *x = agg + ((int64) j * dimensions);
Vector *vec = (Vector *) VectorArrayGet(aggCenters, j);
if (centerCounts[j] > 0)
{
@@ -224,27 +372,27 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
/* TODO Update bounds */
for (int k = 0; k < dimensions; k++)
{
if (isinf(x[k]))
x[k] = x[k] > 0 ? FLT_MAX : -FLT_MAX;
if (isinf(vec->x[k]))
vec->x[k] = vec->x[k] > 0 ? FLT_MAX : -FLT_MAX;
}
for (int k = 0; k < dimensions; k++)
x[k] /= centerCounts[j];
vec->x[k] /= centerCounts[j];
}
else
{
/* TODO Handle empty centers properly */
for (int k = 0; k < dimensions; k++)
x[k] = RandomDouble();
vec->x[k] = RandomDouble();
}
}
/* Set new centers */
UpdateCenters(agg, newCenters, typeInfo);
/* Set new centers if different from agg centers */
SetNewCenters(aggCenters, newCenters, type);
/* Normalize if needed */
if (normprocinfo != NULL)
NormCenters(typeInfo, collation, newCenters);
NormCenters(normalizeprocinfo, collation, newCenters);
}
/*
@@ -256,16 +404,17 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/
static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, IvfflatType type)
{
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
int dimensions = centers->dim;
int numCenters = centers->maxlen;
int numSamples = samples->length;
VectorArray newCenters;
float *agg;
VectorArray aggCenters;
int *centerCounts;
int *closestCenters;
float *lowerBound;
@@ -273,12 +422,14 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
float *s;
float *halfcdist;
float *newcdist;
MemoryContext kmeansCtx;
MemoryContext oldCtx;
/* Calculate allocation sizes */
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->itemsize);
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->itemsize);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
Size aggSize = sizeof(float) * (int64) numCenters * dimensions;
Size aggCentersSize = type == IVFFLAT_TYPE_VECTOR ? 0 : VECTOR_ARRAY_SIZE(numCenters, VECTOR_SIZE(dimensions));
Size centerCountsSize = sizeof(int) * numCenters;
Size closestCentersSize = sizeof(int) * numSamples;
Size lowerBoundSize = sizeof(float) * numSamples * numCenters;
@@ -288,7 +439,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
Size newcdistSize = sizeof(float) * numCenters;
/* Calculate total size */
Size totalSize = samplesSize + centersSize + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
Size totalSize = samplesSize + centersSize + newCentersSize + aggCentersSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
/* Check memory requirements */
/* Add one to error message to ceil */
@@ -305,11 +456,17 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
/* Set support functions */
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
collation = index->rd_indcollation[0];
/* Use memory context */
kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context",
ALLOCSET_DEFAULT_SIZES);
oldCtx = MemoryContextSwitchTo(kmeansCtx);
/* Allocate space */
/* Use float instead of double to save memory */
agg = palloc(aggSize);
centerCounts = palloc(centerCountsSize);
closestCenters = palloc(closestCentersSize);
lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
@@ -318,12 +475,53 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
halfcdist = palloc_extended(halfcdistSize, MCXT_ALLOC_HUGE);
newcdist = palloc(newcdistSize);
/* Initialize new centers */
newCenters = VectorArrayInit(numCenters, dimensions, centers->itemsize);
newCenters->length = numCenters;
aggCenters = VectorArrayInit(numCenters, dimensions, VECTOR_SIZE(dimensions));
aggCenters->length = numCenters;
for (int j = 0; j < numCenters; j++)
{
Vector *vec = (Vector *) VectorArrayGet(aggCenters, j);
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
}
if (type == IVFFLAT_TYPE_VECTOR)
{
/* Use same centers to save memory */
newCenters = aggCenters;
}
else if (type == IVFFLAT_TYPE_HALFVEC)
{
newCenters = VectorArrayInit(numCenters, dimensions, centers->itemsize);
newCenters->length = numCenters;
for (int j = 0; j < numCenters; j++)
{
HalfVector *vec = (HalfVector *) VectorArrayGet(newCenters, j);
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
}
}
else if (type == IVFFLAT_TYPE_BIT)
{
newCenters = VectorArrayInit(numCenters, dimensions, centers->itemsize);
newCenters->length = numCenters;
for (int j = 0; j < numCenters; j++)
{
VarBit *vec = (VarBit *) VectorArrayGet(newCenters, j);
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
}
}
else
elog(ERROR, "Unsupported type");
#ifdef IVFFLAT_MEMORY
ShowMemoryUsage(MemoryContextGetParent(CurrentMemoryContext), totalSize);
ShowMemoryUsage(oldCtx, totalSize);
#endif
/* Pick initial centers */
@@ -460,7 +658,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
}
/* Step 4: For each center c, let m(c) be mean of all points assigned */
ComputeNewCenters(samples, agg, newCenters, centerCounts, closestCenters, normprocinfo, collation, typeInfo);
ComputeNewCenters(samples, aggCenters, newCenters, centerCounts, closestCenters, normprocinfo, normalizeprocinfo, collation, type);
/* Step 5 */
for (int j = 0; j < numCenters; j++)
@@ -491,68 +689,82 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
if (changes == 0 && iteration != 0)
break;
}
}
/*
* Ensure no NaN or infinite values
*/
static void
CheckElements(VectorArray centers, const IvfflatTypeInfo * typeInfo)
{
float *scratch = palloc(sizeof(float) * centers->dim);
for (int i = 0; i < centers->length; i++)
{
for (int j = 0; j < centers->dim; j++)
scratch[j] = 0;
/* /fp:fast may not propagate NaN with MSVC, but that's alright */
typeInfo->sumCenter(VectorArrayGet(centers, i), scratch);
for (int j = 0; j < centers->dim; j++)
{
if (isnan(scratch[j]))
elog(ERROR, "NaN detected. Please report a bug.");
if (isinf(scratch[j]))
elog(ERROR, "Infinite value detected. Please report a bug.");
}
}
}
/*
* Ensure no zero vectors for cosine distance
*/
static void
CheckNorms(VectorArray centers, Relation index)
{
/* Check NORM_PROC instead of KMEANS_NORM_PROC */
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
Oid collation = index->rd_indcollation[0];
if (normprocinfo == NULL)
return;
for (int i = 0; i < centers->length; i++)
{
double norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(VectorArrayGet(centers, i))));
if (norm == 0)
elog(ERROR, "Zero norm detected. Please report a bug.");
}
MemoryContextSwitchTo(oldCtx);
MemoryContextDelete(kmeansCtx);
}
/*
* Detect issues with centers
*/
static void
CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
CheckCenters(Relation index, VectorArray centers, IvfflatType type)
{
FmgrInfo *normprocinfo;
if (centers->length != centers->maxlen)
elog(ERROR, "Not enough centers. Please report a bug.");
CheckElements(centers, typeInfo);
CheckNorms(centers, index);
/* Ensure no NaN or infinite values */
for (int i = 0; i < centers->length; i++)
{
if (type == IVFFLAT_TYPE_VECTOR)
{
Vector *vec = (Vector *) VectorArrayGet(centers, i);
for (int j = 0; j < vec->dim; j++)
{
if (isnan(vec->x[j]))
elog(ERROR, "NaN detected. Please report a bug.");
if (isinf(vec->x[j]))
elog(ERROR, "Infinite value detected. Please report a bug.");
}
}
else if (type == IVFFLAT_TYPE_HALFVEC)
{
HalfVector *vec = (HalfVector *) VectorArrayGet(centers, i);
for (int j = 0; j < vec->dim; j++)
{
if (HalfIsNan(vec->x[j]))
elog(ERROR, "NaN detected. Please report a bug.");
if (HalfIsInf(vec->x[j]))
elog(ERROR, "Infinite value detected. Please report a bug.");
}
}
else if (type != IVFFLAT_TYPE_BIT)
elog(ERROR, "Unsupported type");
}
if (type != IVFFLAT_TYPE_BIT)
{
/* Ensure no duplicate centers */
SortVectorArray(centers, type);
for (int i = 1; i < centers->length; i++)
{
if (datumIsEqual(PointerGetDatum(VectorArrayGet(centers, i)), PointerGetDatum(VectorArrayGet(centers, i - 1)), false, -1))
elog(ERROR, "Duplicate centers detected. Please report a bug.");
}
}
/* Ensure no zero vectors for cosine distance */
/* Check NORM_PROC instead of KMEANS_NORM_PROC */
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
if (normprocinfo != NULL)
{
Oid collation = index->rd_indcollation[0];
for (int i = 0; i < centers->length; i++)
{
double norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(VectorArrayGet(centers, i))));
if (norm == 0)
elog(ERROR, "Zero norm detected. Please report a bug.");
}
}
}
/*
@@ -560,20 +772,12 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
* We use spherical k-means for inner product and cosine
*/
void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, IvfflatType type)
{
MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context",
ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(kmeansCtx);
if (samples->length == 0)
RandomCenters(index, centers, typeInfo);
if (samples->length <= centers->maxlen)
QuickCenters(index, samples, centers, type);
else
ElkanKmeans(index, samples, centers, typeInfo);
ElkanKmeans(index, samples, centers, type);
CheckCenters(index, centers, typeInfo);
MemoryContextSwitchTo(oldCtx);
MemoryContextDelete(kmeansCtx);
CheckCenters(index, centers, type);
}

View File

@@ -11,23 +11,16 @@
#include "pgstat.h"
#include "storage/bufmgr.h"
#ifdef IVFFLAT_MEMORY
#include "utils/memutils.h"
#endif
#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
/*
* Compare list distances
*/
static int
CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{
if (GetScanListConst(a)->distance > GetScanListConst(b)->distance)
if (((const IvfflatScanList *) a)->distance > ((const IvfflatScanList *) b)->distance)
return 1;
if (GetScanListConst(a)->distance < GetScanListConst(b)->distance)
if (((const IvfflatScanList *) a)->distance < ((const IvfflatScanList *) b)->distance)
return -1;
return 0;
@@ -79,14 +72,14 @@ GetScanLists(IndexScanDesc scan, Datum value)
/* Calculate max distance */
if (listCount == so->probes)
maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
else if (distance < maxDistance)
{
IvfflatScanList *scanlist;
/* Remove */
scanlist = GetScanList(pairingheap_remove_first(so->listQueue));
scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
/* Reuse */
scanlist->startPage = list->startPage;
@@ -94,7 +87,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Update max distance */
maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
}
@@ -113,12 +106,19 @@ GetScanItems(IndexScanDesc scan, Datum value)
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
double tuples = 0;
TupleTableSlot *slot = so->vslot;
TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
/*
* Reuse same set of shared buffers for scan
*
* See postgres/src/backend/storage/buffer/README for description
*/
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
/* Search closest probes lists */
while (!pairingheap_is_empty(so->listQueue))
{
BlockNumber searchPage = GetScanList(pairingheap_remove_first(so->listQueue))->startPage;
BlockNumber searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
/* Search all entry pages for list */
while (BlockNumberIsValid(searchPage))
@@ -127,7 +127,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
Page page;
OffsetNumber maxoffno;
buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, so->bas);
buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
@@ -166,6 +166,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
}
}
FreeAccessStrategy(bas);
if (tuples < 100)
ereport(DEBUG1,
(errmsg("index scan found few tuples"),
@@ -207,28 +209,14 @@ GetScanValue(IndexScanDesc scan)
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Normalize if needed */
/* Check normprocinfo since normalizeprocinfo not set for vector */
if (so->normprocinfo != NULL)
value = IvfflatNormValue(so->typeInfo, so->collation, value);
value = IvfflatNormValue(so->normalizeprocinfo, so->collation, value);
}
return value;
}
/*
* Initialize scan sort state
*/
static Tuplesortstate *
InitScanSortState(TupleDesc tupdesc)
{
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
}
/*
* Prepare for an index scan
*/
@@ -239,6 +227,10 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
IvfflatScanOpaque so;
int lists;
int dimensions;
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
int probes = ivfflat_probes;
scan = RelationGetIndexScan(index, nkeys, norderbys);
@@ -250,7 +242,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
probes = lists;
so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + probes * sizeof(IvfflatScanList));
so->typeInfo = IvfflatGetTypeInfo(index);
so->first = true;
so->probes = probes;
so->dimensions = dimensions;
@@ -258,6 +249,7 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
/* Set support functions */
so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
so->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
so->normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
so->collation = index->rd_indcollation[0];
/* Create tuple description for sorting */
@@ -266,18 +258,9 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
/* Prep sort */
so->sortstate = InitScanSortState(so->tupdesc);
so->sortstate = tuplesort_begin_heap(so->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
/* Need separate slots for puttuple and gettuple */
so->vslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
so->mslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
/*
* Reuse same set of shared buffers for scan
*
* See postgres/src/backend/storage/buffer/README for description
*/
so->bas = GetAccessStrategy(BAS_BULKREAD);
so->slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
so->listQueue = pairingheap_allocate(CompareLists, scan);
@@ -294,8 +277,10 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
#if PG_VERSION_NUM >= 130000
if (!so->first)
tuplesort_reset(so->sortstate);
#endif
so->first = true;
pairingheap_reset(so->listQueue);
@@ -342,19 +327,14 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
IvfflatBench("GetScanItems", GetScanItems(scan, value));
so->first = false;
#if defined(IVFFLAT_MEMORY)
elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
#endif
/* Clean up if we allocated a new value */
if (value != scan->orderByData->sk_argument)
pfree(DatumGetPointer(value));
}
if (tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
if (tuplesort_gettupleslot(so->sortstate, true, false, so->slot, NULL))
{
bool isnull;
ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
scan->xs_heaptid = *heaptid;
scan->xs_recheck = false;
@@ -375,10 +355,6 @@ ivfflatendscan(IndexScanDesc scan)
pairingheap_free(so->listQueue);
tuplesort_end(so->sortstate);
FreeAccessStrategy(so->bas);
FreeTupleDesc(so->tupdesc);
/* TODO Free vslot and mslot without freeing TupleDesc */
pfree(so);
scan->opaque = NULL;

View File

@@ -1,11 +1,8 @@
#include "postgres.h"
#include "access/generic_xlog.h"
#include "bitvec.h"
#include "catalog/pg_type.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
@@ -64,13 +61,37 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
return index_getprocinfo(index, 1, procnum);
}
/*
* Get type
*/
IvfflatType
IvfflatGetType(Relation index)
{
FmgrInfo *procinfo = IvfflatOptionalProcInfo(index, IVFFLAT_TYPE_SUPPORT_PROC);
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
IvfflatType result;
if (procinfo == NULL)
return IVFFLAT_TYPE_VECTOR;
result = (IvfflatType) DatumGetInt32(FunctionCall1(procinfo, ObjectIdGetDatum(typid)));
if (result == IVFFLAT_TYPE_UNSUPPORTED)
elog(ERROR, "type not supported for ivfflat index");
return result;
}
/*
* Normalize value
*/
Datum
IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value)
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum value)
{
return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
if (procinfo == NULL)
return DirectFunctionCall1(l2_normalize, value);
return FunctionCall1Coll(procinfo, collation, value);
}
/*
@@ -170,11 +191,7 @@ IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions)
page = BufferGetPage(buf);
metap = IvfflatPageGetMeta(page);
if (unlikely(metap->magicNumber != IVFFLAT_MAGIC_NUMBER))
elog(ERROR, "ivfflat index is not valid");
if (lists != NULL)
*lists = metap->lists;
*lists = metap->lists;
if (dimensions != NULL)
*dimensions = metap->dimensions;
@@ -229,145 +246,21 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
}
}
PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
static Size
VectorItemSize(int dimensions)
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ivfflat_support);
Datum
halfvec_ivfflat_support(PG_FUNCTION_ARGS)
{
return VECTOR_SIZE(dimensions);
}
PG_RETURN_INT32(IVFFLAT_TYPE_HALFVEC);
};
static Size
HalfvecItemSize(int dimensions)
PGDLLEXPORT PG_FUNCTION_INFO_V1(bit_ivfflat_support);
Datum
bit_ivfflat_support(PG_FUNCTION_ARGS)
{
return HALFVEC_SIZE(dimensions);
}
Oid typid = PG_GETARG_OID(0);
static Size
BitItemSize(int dimensions)
{
return VARBITTOTALLEN(dimensions);
}
static void
VectorUpdateCenter(Pointer v, int dimensions, float *x)
{
Vector *vec = (Vector *) v;
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
for (int k = 0; k < dimensions; k++)
vec->x[k] = x[k];
}
static void
HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
{
HalfVector *vec = (HalfVector *) v;
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
for (int k = 0; k < dimensions; k++)
vec->x[k] = Float4ToHalfUnchecked(x[k]);
}
static void
BitUpdateCenter(Pointer v, int dimensions, float *x)
{
VarBit *vec = (VarBit *) v;
unsigned char *nx = VARBITS(vec);
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
for (uint32 k = 0; k < VARBITBYTES(vec); k++)
nx[k] = 0;
for (int k = 0; k < dimensions; k++)
nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
}
static void
VectorSumCenter(Pointer v, float *x)
{
Vector *vec = (Vector *) v;
for (int k = 0; k < vec->dim; k++)
x[k] += vec->x[k];
}
static void
HalfvecSumCenter(Pointer v, float *x)
{
HalfVector *vec = (HalfVector *) v;
for (int k = 0; k < vec->dim; k++)
x[k] += HalfToFloat4(vec->x[k]);
}
static void
BitSumCenter(Pointer v, float *x)
{
VarBit *vec = (VarBit *) v;
for (int k = 0; k < VARBITLEN(vec); k++)
x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
}
/*
* Get type info
*/
const IvfflatTypeInfo *
IvfflatGetTypeInfo(Relation index)
{
FmgrInfo *procinfo = IvfflatOptionalProcInfo(index, IVFFLAT_TYPE_INFO_PROC);
if (procinfo == NULL)
{
static const IvfflatTypeInfo typeInfo = {
.maxDimensions = IVFFLAT_MAX_DIM,
.normalize = l2_normalize,
.itemSize = VectorItemSize,
.updateCenter = VectorUpdateCenter,
.sumCenter = VectorSumCenter
};
return (&typeInfo);
}
if (typid == BITOID)
PG_RETURN_INT32(IVFFLAT_TYPE_BIT);
else
return (const IvfflatTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid));
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_halfvec_support);
Datum
ivfflat_halfvec_support(PG_FUNCTION_ARGS)
{
static const IvfflatTypeInfo typeInfo = {
.maxDimensions = IVFFLAT_MAX_DIM * 2,
.normalize = halfvec_l2_normalize,
.itemSize = HalfvecItemSize,
.updateCenter = HalfvecUpdateCenter,
.sumCenter = HalfvecSumCenter
};
PG_RETURN_POINTER(&typeInfo);
};
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
Datum
ivfflat_bit_support(PG_FUNCTION_ARGS)
{
static const IvfflatTypeInfo typeInfo = {
.maxDimensions = IVFFLAT_MAX_DIM * 32,
.normalize = NULL,
.itemSize = BitItemSize,
.updateCenter = BitUpdateCenter,
.sumCenter = BitSumCenter
};
PG_RETURN_POINTER(&typeInfo);
PG_RETURN_INT32(IVFFLAT_TYPE_UNSUPPORTED);
};

View File

@@ -3,8 +3,6 @@
#include <limits.h>
#include <math.h>
#include "catalog/pg_type.h"
#include "common/string.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
@@ -12,7 +10,6 @@
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "vector.h"
#if PG_VERSION_NUM >= 120000
@@ -100,24 +97,27 @@ CheckIndex(int32 *indices, int i, int dim)
{
int32 index = indices[i];
if (index < 0 || index >= dim)
{
if (index < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec index out of bounds")));
}
errmsg("index must be greater than zero")));
if (index > dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("index must be less than or equal to dimensions")));
if (i > 0)
{
if (index < indices[i - 1])
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec indices must be in ascending order")));
errmsg("indexes must be in ascending order")));
if (index == indices[i - 1])
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec indices must not contain duplicates")));
errmsg("indexes must not contain duplicates")));
}
}
@@ -190,13 +190,13 @@ CompareIndices(const void *a, const void *b)
/*
* Convert textual representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_in);
Datum
sparsevec_in(PG_FUNCTION_ARGS)
{
char *lit = PG_GETARG_CSTRING(0);
int32 typmod = PG_GETARG_INT32(2);
long dim;
int dim;
char *pt = lit;
char *stringEnd;
SparseVector *result;
@@ -246,6 +246,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
long index;
float value;
/* TODO Better error */
if (nnz == maxNnz)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
@@ -261,6 +262,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Use similar logic as int2vectorin */
errno = 0;
index = strtol(pt, &stringEnd, 10);
if (stringEnd == pt)
@@ -268,11 +270,10 @@ sparsevec_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Keep in int range for correct error message later */
if (index > INT_MAX)
index = INT_MAX;
else if (index < INT_MIN + 1)
index = INT_MIN + 1;
if (errno == ERANGE || index < 1 || index > INT_MAX)
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("index \"%ld\" is out of range for type sparsevec", index)));
pt = stringEnd;
@@ -311,8 +312,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
/* Do not store zero values */
if (value != 0)
{
/* Convert 1-based numbering (SQL) to 0-based (C) */
elements[nnz].index = index - 1;
elements[nnz].index = index;
elements[nnz].value = value;
nnz++;
}
@@ -351,6 +351,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
pt++;
/* Use similar logic as int2vectorin */
errno = 0;
dim = strtol(pt, &stringEnd, 10);
if (stringEnd == pt)
@@ -358,12 +359,6 @@ sparsevec_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Keep in int range for correct error message later */
if (dim > INT_MAX)
dim = INT_MAX;
else if (dim < INT_MIN)
dim = INT_MIN;
pt = stringEnd;
/* Only whitespace is allowed after the closing brace */
@@ -411,7 +406,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
/*
* Convert internal representation to textual representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_out);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_out);
Datum
sparsevec_out(PG_FUNCTION_ARGS)
{
@@ -446,8 +441,7 @@ sparsevec_out(PG_FUNCTION_ARGS)
if (i > 0)
AppendChar(ptr, ',');
/* Convert 0-based numbering (C) to 1-based (SQL) */
AppendInt(ptr, sparsevec->indices[i] + 1);
AppendInt(ptr, sparsevec->indices[i]);
AppendChar(ptr, ':');
AppendFloat(ptr, values[i]);
}
@@ -464,7 +458,7 @@ sparsevec_out(PG_FUNCTION_ARGS)
/*
* Convert type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
Datum
sparsevec_typmod_in(PG_FUNCTION_ARGS)
{
@@ -495,7 +489,7 @@ sparsevec_typmod_in(PG_FUNCTION_ARGS)
/*
* Convert external binary representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_recv);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_recv);
Datum
sparsevec_recv(PG_FUNCTION_ARGS)
{
@@ -523,7 +517,6 @@ sparsevec_recv(PG_FUNCTION_ARGS)
result = InitSparseVector(dim, nnz);
values = SPARSEVEC_VALUES(result);
/* Binary representation uses zero-based numbering for indices */
for (int i = 0; i < nnz; i++)
{
result->indices[i] = pq_getmsgint(buf, sizeof(int32));
@@ -534,7 +527,6 @@ sparsevec_recv(PG_FUNCTION_ARGS)
{
values[i] = pq_getmsgfloat4(buf);
CheckElement(values[i]);
if (values[i] == 0)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
@@ -547,7 +539,7 @@ sparsevec_recv(PG_FUNCTION_ARGS)
/*
* Convert internal representation to the external binary representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_send);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_send);
Datum
sparsevec_send(PG_FUNCTION_ARGS)
{
@@ -559,11 +551,8 @@ sparsevec_send(PG_FUNCTION_ARGS)
pq_sendint(&buf, svec->dim, sizeof(int32));
pq_sendint(&buf, svec->nnz, sizeof(int32));
pq_sendint(&buf, svec->unused, sizeof(int32));
/* Binary representation uses zero-based numbering for indices */
for (int i = 0; i < svec->nnz; i++)
pq_sendint(&buf, svec->indices[i], sizeof(int32));
for (int i = 0; i < svec->nnz; i++)
pq_sendfloat4(&buf, values[i]);
@@ -574,7 +563,7 @@ sparsevec_send(PG_FUNCTION_ARGS)
* Convert sparse vector to sparse vector
* This is needed to check the type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec);
Datum
sparsevec(PG_FUNCTION_ARGS)
{
@@ -589,7 +578,7 @@ sparsevec(PG_FUNCTION_ARGS)
/*
* Convert dense vector to sparse vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_sparsevec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_sparsevec);
Datum
vector_to_sparsevec(PG_FUNCTION_ARGS)
{
@@ -620,7 +609,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
if (j >= result->nnz)
elog(ERROR, "safety check failed");
result->indices[j] = i;
result->indices[j] = i + 1;
values[j] = vec->x[i];
j++;
}
@@ -632,7 +621,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
/*
* Convert half vector to sparse vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_sparsevec);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_sparsevec);
Datum
halfvec_to_sparsevec(PG_FUNCTION_ARGS)
{
@@ -663,7 +652,7 @@ halfvec_to_sparsevec(PG_FUNCTION_ARGS)
if (j >= result->nnz)
elog(ERROR, "safety check failed");
result->indices[j] = i;
result->indices[j] = i + 1;
values[j] = HalfToFloat4(vec->x[i]);
j++;
}
@@ -672,137 +661,6 @@ halfvec_to_sparsevec(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Convert array to sparse vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_sparsevec);
Datum
array_to_sparsevec(PG_FUNCTION_ARGS)
{
ArrayType *array = PG_GETARG_ARRAYTYPE_P(0);
int32 typmod = PG_GETARG_INT32(1);
SparseVector *result;
int16 typlen;
bool typbyval;
char typalign;
Datum *elemsp;
int nelemsp;
int nnz = 0;
float *values;
int j = 0;
if (ARR_NDIM(array) > 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("array must be 1-D")));
if (ARR_HASNULL(array) && array_contains_nulls(array))
ereport(ERROR,
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
errmsg("array must not contain nulls")));
get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, NULL, &nelemsp);
CheckDim(nelemsp);
CheckExpectedDim(typmod, nelemsp);
#ifdef _MSC_VER
/* /fp:fast may not propagate +/-Infinity or NaN */
#define IS_NOT_ZERO(v) (isnan((float) (v)) || isinf((float) (v)) || ((float) (v)) != 0)
#else
#define IS_NOT_ZERO(v) (((float) (v)) != 0)
#endif
if (ARR_ELEMTYPE(array) == INT4OID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DatumGetInt32(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT8OID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DatumGetFloat8(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT4OID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DatumGetFloat4(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == NUMERICOID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DirectFunctionCall1(numeric_float4, elemsp[i]));
}
else
{
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("unsupported array type")));
}
result = InitSparseVector(nelemsp, nnz);
values = SPARSEVEC_VALUES(result);
#define PROCESS_ARRAY_ELEM(elem) \
do { \
float v = (float) (elem); \
if (IS_NOT_ZERO(v)) { \
/* Safety check */ \
if (j >= result->nnz) \
elog(ERROR, "safety check failed"); \
result->indices[j] = i; \
values[j] = v; \
j++; \
} \
} while (0)
if (ARR_ELEMTYPE(array) == INT4OID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetInt32(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT8OID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetFloat8(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT4OID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetFloat4(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == NUMERICOID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetFloat4(DirectFunctionCall1(numeric_float4, elemsp[i])));
}
else
{
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("unsupported array type")));
}
#undef PROCESS_ARRAY_ELEM
#undef IS_NOT_ZERO
/*
* Free allocation from deconstruct_array. Do not free individual elements
* when pass-by-reference since they point to original array.
*/
pfree(elemsp);
if (j != result->nnz)
elog(ERROR, "correctness check failed");
/* Check elements */
for (int i = 0; i < result->nnz; i++)
CheckElement(values[i]);
PG_RETURN_POINTER(result);
}
/*
* Get the L2 squared distance between sparse vectors
*/
@@ -854,7 +712,7 @@ SparsevecL2SquaredDistance(SparseVector * a, SparseVector * b)
/*
* Get the L2 distance between sparse vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_distance);
Datum
sparsevec_l2_distance(PG_FUNCTION_ARGS)
{
@@ -870,7 +728,7 @@ sparsevec_l2_distance(PG_FUNCTION_ARGS)
* Get the L2 squared distance between sparse vectors
* This saves a sqrt calculation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance);
Datum
sparsevec_l2_squared_distance(PG_FUNCTION_ARGS)
{
@@ -921,7 +779,7 @@ SparsevecInnerProduct(SparseVector * a, SparseVector * b)
/*
* Get the inner product of two sparse vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_inner_product);
Datum
sparsevec_inner_product(PG_FUNCTION_ARGS)
{
@@ -936,7 +794,7 @@ sparsevec_inner_product(PG_FUNCTION_ARGS)
/*
* Get the negative inner product of two sparse vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product);
Datum
sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
{
@@ -951,7 +809,7 @@ sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
/*
* Get the cosine distance between two sparse vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_cosine_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cosine_distance);
Datum
sparsevec_cosine_distance(PG_FUNCTION_ARGS)
{
@@ -996,7 +854,7 @@ sparsevec_cosine_distance(PG_FUNCTION_ARGS)
/*
* Get the L1 distance between two sparse vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l1_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l1_distance);
Datum
sparsevec_l1_distance(PG_FUNCTION_ARGS)
{
@@ -1045,7 +903,7 @@ sparsevec_l1_distance(PG_FUNCTION_ARGS)
/*
* Get the L2 norm of a sparse vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_norm);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_norm);
Datum
sparsevec_l2_norm(PG_FUNCTION_ARGS)
{
@@ -1063,7 +921,7 @@ sparsevec_l2_norm(PG_FUNCTION_ARGS)
/*
* Normalize a sparse vector with the L2 norm
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_normalize);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_normalize);
Datum
sparsevec_l2_normalize(PG_FUNCTION_ARGS)
{
@@ -1155,10 +1013,11 @@ sparsevec_cmp_internal(SparseVector * a, SparseVector * b)
return 1;
}
if (a->nnz < b->nnz && b->indices[nnz] < a->dim)
/* Check <= dim since indices start at 1 */
if (a->nnz < b->nnz && b->indices[nnz] <= a->dim)
return bx[nnz] < 0 ? 1 : -1;
if (a->nnz > b->nnz && a->indices[nnz] < b->dim)
if (a->nnz > b->nnz && a->indices[nnz] <= b->dim)
return ax[nnz] < 0 ? -1 : 1;
if (a->dim < b->dim)
@@ -1173,7 +1032,7 @@ sparsevec_cmp_internal(SparseVector * a, SparseVector * b)
/*
* Less than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_lt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_lt);
Datum
sparsevec_lt(PG_FUNCTION_ARGS)
{
@@ -1186,7 +1045,7 @@ sparsevec_lt(PG_FUNCTION_ARGS)
/*
* Less than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_le);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_le);
Datum
sparsevec_le(PG_FUNCTION_ARGS)
{
@@ -1199,7 +1058,7 @@ sparsevec_le(PG_FUNCTION_ARGS)
/*
* Equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_eq);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_eq);
Datum
sparsevec_eq(PG_FUNCTION_ARGS)
{
@@ -1212,7 +1071,7 @@ sparsevec_eq(PG_FUNCTION_ARGS)
/*
* Not equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_ne);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_ne);
Datum
sparsevec_ne(PG_FUNCTION_ARGS)
{
@@ -1225,7 +1084,7 @@ sparsevec_ne(PG_FUNCTION_ARGS)
/*
* Greater than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_ge);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_ge);
Datum
sparsevec_ge(PG_FUNCTION_ARGS)
{
@@ -1238,7 +1097,7 @@ sparsevec_ge(PG_FUNCTION_ARGS)
/*
* Greater than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_gt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_gt);
Datum
sparsevec_gt(PG_FUNCTION_ARGS)
{
@@ -1251,7 +1110,7 @@ sparsevec_gt(PG_FUNCTION_ARGS)
/*
* Compare sparse vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_cmp);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cmp);
Datum
sparsevec_cmp(PG_FUNCTION_ARGS)
{

View File

@@ -1,23 +1,23 @@
#ifndef SPARSEVEC_H
#define SPARSEVEC_H
#define SPARSEVEC_MAX_DIM 1000000000
#include "fmgr.h"
#define SPARSEVEC_MAX_DIM 1000000
#define SPARSEVEC_MAX_NNZ 16000
#define DatumGetSparseVector(x) ((SparseVector *) PG_DETOAST_DATUM(x))
#define PG_GETARG_SPARSEVEC_P(x) DatumGetSparseVector(PG_GETARG_DATUM(x))
#define PG_RETURN_SPARSEVEC_P(x) PG_RETURN_POINTER(x)
/*
* Indices use 0-based numbering for the on-disk (and binary) format (consistent with C)
* and are always sorted. Values come after indices.
*/
/* Indices are always sorted */
/* Values come after indices */
typedef struct SparseVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int32 dim; /* number of dimensions */
int32 nnz; /* number of non-zero elements */
int32 unused; /* reserved for future use, always zero */
int32 unused;
int32 indices[FLEXIBLE_ARRAY_MEMBER];
} SparseVector;

View File

@@ -26,6 +26,11 @@
#include "varatt.h"
#endif
#if PG_VERSION_NUM < 130000
#define TYPALIGN_DOUBLE 'd'
#define TYPALIGN_INT 'i'
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -155,10 +160,28 @@ CheckStateArray(ArrayType *statearray, const char *caller)
return (float8 *) ARR_DATA_PTR(statearray);
}
#if PG_VERSION_NUM < 120003
static pg_noinline void
float_overflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: overflow")));
}
static pg_noinline void
float_underflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: underflow")));
}
#endif
/*
* Convert textual representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
Datum
vector_in(PG_FUNCTION_ARGS)
{
@@ -271,7 +294,7 @@ vector_in(PG_FUNCTION_ARGS)
/*
* Convert internal representation to textual representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_out);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
Datum
vector_out(PG_FUNCTION_ARGS)
{
@@ -325,7 +348,7 @@ PrintVector(char *msg, Vector * vector)
/*
* Convert type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_typmod_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
Datum
vector_typmod_in(PG_FUNCTION_ARGS)
{
@@ -356,7 +379,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
/*
* Convert external binary representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_recv);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
Datum
vector_recv(PG_FUNCTION_ARGS)
{
@@ -390,7 +413,7 @@ vector_recv(PG_FUNCTION_ARGS)
/*
* Convert internal representation to the external binary representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_send);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
Datum
vector_send(PG_FUNCTION_ARGS)
{
@@ -410,7 +433,7 @@ vector_send(PG_FUNCTION_ARGS)
* Convert vector to vector
* This is needed to check the type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
Datum
vector(PG_FUNCTION_ARGS)
{
@@ -425,7 +448,7 @@ vector(PG_FUNCTION_ARGS)
/*
* Convert array to vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
Datum
array_to_vector(PG_FUNCTION_ARGS)
{
@@ -499,7 +522,7 @@ array_to_vector(PG_FUNCTION_ARGS)
/*
* Convert vector to float4[]
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_float4);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
Datum
vector_to_float4(PG_FUNCTION_ARGS)
{
@@ -523,7 +546,7 @@ vector_to_float4(PG_FUNCTION_ARGS)
/*
* Convert half vector to vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_vector);
Datum
halfvec_to_vector(PG_FUNCTION_ARGS)
{
@@ -561,7 +584,7 @@ VectorL2SquaredDistance(int dim, float *ax, float *bx)
/*
* Get the L2 distance between vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
Datum
l2_distance(PG_FUNCTION_ARGS)
{
@@ -577,7 +600,7 @@ l2_distance(PG_FUNCTION_ARGS)
* Get the L2 squared distance between vectors
* This saves a sqrt calculation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
Datum
vector_l2_squared_distance(PG_FUNCTION_ARGS)
{
@@ -604,7 +627,7 @@ VectorInnerProduct(int dim, float *ax, float *bx)
/*
* Get the inner product of two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
Datum
inner_product(PG_FUNCTION_ARGS)
{
@@ -619,7 +642,7 @@ inner_product(PG_FUNCTION_ARGS)
/*
* Get the negative inner product of two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_negative_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
Datum
vector_negative_inner_product(PG_FUNCTION_ARGS)
{
@@ -653,7 +676,7 @@ VectorCosineSimilarity(int dim, float *ax, float *bx)
/*
* Get the cosine distance between two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(cosine_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
Datum
cosine_distance(PG_FUNCTION_ARGS)
{
@@ -685,7 +708,7 @@ cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm)
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_spherical_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
Datum
vector_spherical_distance(PG_FUNCTION_ARGS)
{
@@ -722,7 +745,7 @@ VectorL1Distance(int dim, float *ax, float *bx)
/*
* Get the L1 distance between two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l1_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(l1_distance);
Datum
l1_distance(PG_FUNCTION_ARGS)
{
@@ -737,7 +760,7 @@ l1_distance(PG_FUNCTION_ARGS)
/*
* Get the dimensions of a vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_dims);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
Datum
vector_dims(PG_FUNCTION_ARGS)
{
@@ -749,7 +772,7 @@ vector_dims(PG_FUNCTION_ARGS)
/*
* Get the L2 norm of a vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_norm);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
Datum
vector_norm(PG_FUNCTION_ARGS)
{
@@ -767,7 +790,7 @@ vector_norm(PG_FUNCTION_ARGS)
/*
* Normalize a vector with the L2 norm
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_normalize);
PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_normalize);
Datum
l2_normalize(PG_FUNCTION_ARGS)
{
@@ -806,7 +829,7 @@ l2_normalize(PG_FUNCTION_ARGS)
/*
* Add vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_add);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
Datum
vector_add(PG_FUNCTION_ARGS)
{
@@ -839,7 +862,7 @@ vector_add(PG_FUNCTION_ARGS)
/*
* Subtract vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_sub);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
Datum
vector_sub(PG_FUNCTION_ARGS)
{
@@ -872,7 +895,7 @@ vector_sub(PG_FUNCTION_ARGS)
/*
* Multiply vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_mul);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_mul);
Datum
vector_mul(PG_FUNCTION_ARGS)
{
@@ -908,7 +931,7 @@ vector_mul(PG_FUNCTION_ARGS)
/*
* Concatenate vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_concat);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_concat);
Datum
vector_concat(PG_FUNCTION_ARGS)
{
@@ -932,7 +955,7 @@ vector_concat(PG_FUNCTION_ARGS)
/*
* Quantize a vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(binary_quantize);
PGDLLEXPORT PG_FUNCTION_INFO_V1(binary_quantize);
Datum
binary_quantize(PG_FUNCTION_ARGS)
{
@@ -950,39 +973,24 @@ binary_quantize(PG_FUNCTION_ARGS)
/*
* Get a subvector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(subvector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
Datum
subvector(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
int32 start = PG_GETARG_INT32(1);
int32 count = PG_GETARG_INT32(2);
int32 end;
int32 end = start + count;
float *ax = a->x;
Vector *result;
int dim;
if (count < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension")));
/*
* Check if (start + count > a->dim), avoiding integer overflow. a->dim
* and count are both positive, so a->dim - count won't overflow.
*/
if (start > a->dim - count)
end = a->dim + 1;
else
end = start + count;
/* Indexing starts at 1, like substring */
if (start < 1)
start = 1;
else if (start > a->dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension")));
if (end > a->dim)
end = a->dim + 1;
dim = end - start;
CheckDim(dim);
@@ -1024,7 +1032,7 @@ vector_cmp_internal(Vector * a, Vector * b)
/*
* Less than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_lt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
Datum
vector_lt(PG_FUNCTION_ARGS)
{
@@ -1037,7 +1045,7 @@ vector_lt(PG_FUNCTION_ARGS)
/*
* Less than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_le);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
Datum
vector_le(PG_FUNCTION_ARGS)
{
@@ -1050,7 +1058,7 @@ vector_le(PG_FUNCTION_ARGS)
/*
* Equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_eq);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
Datum
vector_eq(PG_FUNCTION_ARGS)
{
@@ -1063,7 +1071,7 @@ vector_eq(PG_FUNCTION_ARGS)
/*
* Not equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ne);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
Datum
vector_ne(PG_FUNCTION_ARGS)
{
@@ -1076,7 +1084,7 @@ vector_ne(PG_FUNCTION_ARGS)
/*
* Greater than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ge);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
Datum
vector_ge(PG_FUNCTION_ARGS)
{
@@ -1089,7 +1097,7 @@ vector_ge(PG_FUNCTION_ARGS)
/*
* Greater than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_gt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
Datum
vector_gt(PG_FUNCTION_ARGS)
{
@@ -1102,7 +1110,7 @@ vector_gt(PG_FUNCTION_ARGS)
/*
* Compare vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_cmp);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
Datum
vector_cmp(PG_FUNCTION_ARGS)
{
@@ -1115,7 +1123,7 @@ vector_cmp(PG_FUNCTION_ARGS)
/*
* Accumulate vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_accum);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_accum);
Datum
vector_accum(PG_FUNCTION_ARGS)
{
@@ -1174,13 +1182,12 @@ vector_accum(PG_FUNCTION_ARGS)
}
/*
* Combine vectors or half vectors (also used for halfvec_combine)
* Combine vectors or half vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_combine);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_combine);
Datum
vector_combine(PG_FUNCTION_ARGS)
{
/* Must also update parameters of halfvec_combine if modifying */
ArrayType *statearray1 = PG_GETARG_ARRAYTYPE_P(0);
ArrayType *statearray2 = PG_GETARG_ARRAYTYPE_P(1);
float8 *statevalues1;
@@ -1247,7 +1254,7 @@ vector_combine(PG_FUNCTION_ARGS)
/*
* Average vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_avg);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_avg);
Datum
vector_avg(PG_FUNCTION_ARGS)
{
@@ -1281,7 +1288,7 @@ vector_avg(PG_FUNCTION_ARGS)
/*
* Convert sparse vector to dense vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_vector);
Datum
sparsevec_to_vector(PG_FUNCTION_ARGS)
{
@@ -1296,7 +1303,7 @@ sparsevec_to_vector(PG_FUNCTION_ARGS)
result = InitVector(dim);
for (int i = 0; i < svec->nnz; i++)
result->x[svec->indices[i]] = values[i];
result->x[svec->indices[i] - 1] = values[i];
PG_RETURN_POINTER(result);
}

View File

@@ -1,6 +1,8 @@
#ifndef VECTOR_H
#define VECTOR_H
#include "fmgr.h"
#define VECTOR_MAX_DIM 16000
#define VECTOR_SIZE(_dim) (offsetof(Vector, x) + sizeof(float)*(_dim))
@@ -12,19 +14,13 @@ typedef struct Vector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int16 dim; /* number of dimensions */
int16 unused; /* reserved for future use, always zero */
int16 unused;
float x[FLEXIBLE_ARRAY_MEMBER];
} Vector;
Vector *InitVector(int dim);
void PrintVector(char *msg, Vector * vector);
int vector_cmp_internal(Vector * a, Vector * b);
/* TODO Move to better place */
#if PG_VERSION_NUM >= 160000
#define FUNCTION_PREFIX
#else
#define FUNCTION_PREFIX PGDLLEXPORT
#endif
PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
#endif

View File

@@ -1,61 +0,0 @@
SET enable_seqscan = off;
-- vector
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
val
---------
[1,2,3]
(1 row)
SELECT * FROM t ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
val
---------
[1,2,3]
(1 row)
SELECT * FROM t ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '{1:1,2:2,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
(1 row)
SELECT * FROM t ORDER BY val;
val
-----------------
{}/3
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
(4 rows)
DROP TABLE t;

View File

@@ -0,0 +1,20 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
val
---------
[1,2,3]
(1 row)
SELECT * FROM t ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;

View File

@@ -0,0 +1,20 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '{1:1,2:2,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
(1 row)
SELECT * FROM t ORDER BY val;
val
-----------------
{}/3
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
(4 rows)
DROP TABLE t;

View File

@@ -0,0 +1,20 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
val
---------
[1,2,3]
(1 row)
SELECT * FROM t ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;

View File

@@ -208,62 +208,6 @@ SELECT '{1:1e-8}/1'::sparsevec::halfvec;
[0]
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
sparsevec
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
sparsevec
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
ERROR: expected 5 dimensions, not 6
SELECT '{NULL}'::real[]::sparsevec;
ERROR: array must not contain nulls
SELECT '{NaN}'::real[]::sparsevec;
ERROR: NaN not allowed in sparsevec
SELECT '{Infinity}'::real[]::sparsevec;
ERROR: infinite value not allowed in sparsevec
SELECT '{-Infinity}'::real[]::sparsevec;
ERROR: infinite value not allowed in sparsevec
SELECT '{}'::real[]::sparsevec;
ERROR: sparsevec must have at least 1 dimension
SELECT '{{1}}'::real[]::sparsevec;
ERROR: array must be 1-D
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
ERROR: vector cannot have more than 16000 dimensions
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -1,50 +1,15 @@
-- vector
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3));
\copy t TO 'results/vector.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/vector.bin' WITH (FORMAT binary)
CREATE TABLE t (val vector(3), val2 halfvec(3), val3 sparsevec(3));
INSERT INTO t (val, val2, val3) VALUES ('[0,0,0]', '[0,0,0]', '{}/3'), ('[1,2,3]', '[1,2,3]', '{1:1,2:2,3:3}/3'), ('[1,1,1]', '[1,1,1]', '{1:1,2:1,3:1}/3'), (NULL, NULL, NULL);
CREATE TABLE t2 (val vector(3), val2 halfvec(3), val3 sparsevec(3));
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
DROP TABLE t2;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val halfvec(3));
\copy t TO 'results/halfvec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/halfvec.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
DROP TABLE t2;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE TABLE t2 (val sparsevec(3));
\copy t TO 'results/sparsevec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/sparsevec.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
-----------------
{}/3
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
val | val2 | val3
---------+---------+-----------------
[0,0,0] | [0,0,0] | {}/3
[1,1,1] | [1,1,1] | {1:1,2:1,3:1}/3
[1,2,3] | [1,2,3] | {1:1,2:2,3:3}/3
| |
(4 rows)
DROP TABLE t;

View File

@@ -1,167 +1,3 @@
SELECT '[1,2,3]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::halfvec;
halfvec
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::halfvec;
halfvec
------------
[1.234375]
(1 row)
SELECT '[hello,1]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[hello,1]"
LINE 1: SELECT '[hello,1]'::halfvec;
^
SELECT '[NaN,1]'::halfvec;
ERROR: NaN not allowed in halfvec
LINE 1: SELECT '[NaN,1]'::halfvec;
^
SELECT '[Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[Infinity,1]'::halfvec;
^
SELECT '[-Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[-Infinity,1]'::halfvec;
^
SELECT '[65519,-65519]'::halfvec;
halfvec
----------------
[65504,-65504]
(1 row)
SELECT '[65520,-65520]'::halfvec;
ERROR: "65520" is out of range for type halfvec
LINE 1: SELECT '[65520,-65520]'::halfvec;
^
SELECT '[1e-8,-1e-8]'::halfvec;
halfvec
---------
[0,-0]
(1 row)
SELECT '[4e38,1]'::halfvec;
ERROR: "4e38" is out of range for type halfvec
LINE 1: SELECT '[4e38,1]'::halfvec;
^
SELECT '[1e-46,1]'::halfvec;
halfvec
---------
[0,1]
(1 row)
SELECT '[1,2,3'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,2,3"
LINE 1: SELECT '[1,2,3'::halfvec;
^
SELECT '[1,2,3]9'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::halfvec;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::halfvec;
ERROR: invalid input syntax for type halfvec: "1,2,3"
LINE 1: SELECT '1,2,3'::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT ''::halfvec;
ERROR: invalid input syntax for type halfvec: ""
LINE 1: SELECT ''::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT '['::halfvec;
ERROR: invalid input syntax for type halfvec: "["
LINE 1: SELECT '['::halfvec;
^
SELECT '[ '::halfvec;
ERROR: invalid input syntax for type halfvec: "[ "
LINE 1: SELECT '[ '::halfvec;
^
SELECT '[,'::halfvec;
ERROR: invalid input syntax for type halfvec: "[,"
LINE 1: SELECT '[,'::halfvec;
^
SELECT '[]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[]'::halfvec;
^
SELECT '[ ]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[ ]'::halfvec;
^
SELECT '[,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[,]"
LINE 1: SELECT '[,]'::halfvec;
^
SELECT '[1,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,]"
LINE 1: SELECT '[1,]'::halfvec;
^
SELECT '[1a]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1a]"
LINE 1: SELECT '[1a]'::halfvec;
^
SELECT '[1,,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,,3]"
LINE 1: SELECT '[1,,3]'::halfvec;
^
SELECT '[1, ,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::halfvec;
^
SELECT '[1,2,3]'::halfvec(3);
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::halfvec(3, 2);
^
SELECT '[1,2,3]'::halfvec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::halfvec('a');
^
SELECT '[1,2,3]'::halfvec(0);
ERROR: dimensions for type halfvec must be at least 1
LINE 1: SELECT '[1,2,3]'::halfvec(0);
^
SELECT '[1,2,3]'::halfvec(16001);
ERROR: dimensions for type halfvec cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::halfvec(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::halfvec(2)[];
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec + '[4,5,6]';
?column?
----------
@@ -570,20 +406,6 @@ SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, 2147483647, 10);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2147483647);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, -2147483644, 2147483647);
subvector
-----------
[1,2]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]']) v;
avg
-----------

View File

@@ -0,0 +1,164 @@
SELECT '[1,2,3]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::halfvec;
halfvec
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::halfvec;
halfvec
------------
[1.234375]
(1 row)
SELECT '[hello,1]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[hello,1]"
LINE 1: SELECT '[hello,1]'::halfvec;
^
SELECT '[NaN,1]'::halfvec;
ERROR: NaN not allowed in halfvec
LINE 1: SELECT '[NaN,1]'::halfvec;
^
SELECT '[Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[Infinity,1]'::halfvec;
^
SELECT '[-Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[-Infinity,1]'::halfvec;
^
SELECT '[65519,-65519]'::halfvec;
halfvec
----------------
[65504,-65504]
(1 row)
SELECT '[65520,-65520]'::halfvec;
ERROR: "65520" is out of range for type halfvec
LINE 1: SELECT '[65520,-65520]'::halfvec;
^
SELECT '[1e-8,-1e-8]'::halfvec;
halfvec
---------
[0,-0]
(1 row)
SELECT '[4e38,1]'::halfvec;
ERROR: "4e38" is out of range for type halfvec
LINE 1: SELECT '[4e38,1]'::halfvec;
^
SELECT '[1e-46,1]'::halfvec;
halfvec
---------
[0,1]
(1 row)
SELECT '[1,2,3'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,2,3"
LINE 1: SELECT '[1,2,3'::halfvec;
^
SELECT '[1,2,3]9'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::halfvec;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::halfvec;
ERROR: invalid input syntax for type halfvec: "1,2,3"
LINE 1: SELECT '1,2,3'::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT ''::halfvec;
ERROR: invalid input syntax for type halfvec: ""
LINE 1: SELECT ''::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT '['::halfvec;
ERROR: invalid input syntax for type halfvec: "["
LINE 1: SELECT '['::halfvec;
^
SELECT '[ '::halfvec;
ERROR: invalid input syntax for type halfvec: "[ "
LINE 1: SELECT '[ '::halfvec;
^
SELECT '[,'::halfvec;
ERROR: invalid input syntax for type halfvec: "[,"
LINE 1: SELECT '[,'::halfvec;
^
SELECT '[]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[]'::halfvec;
^
SELECT '[ ]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[ ]'::halfvec;
^
SELECT '[,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[,]"
LINE 1: SELECT '[,]'::halfvec;
^
SELECT '[1,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,]"
LINE 1: SELECT '[1,]'::halfvec;
^
SELECT '[1a]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1a]"
LINE 1: SELECT '[1a]'::halfvec;
^
SELECT '[1,,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,,3]"
LINE 1: SELECT '[1,,3]'::halfvec;
^
SELECT '[1, ,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::halfvec;
^
SELECT '[1,2,3]'::halfvec(3);
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::halfvec(3, 2);
^
SELECT '[1,2,3]'::halfvec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::halfvec('a');
^
SELECT '[1,2,3]'::halfvec(0);
ERROR: dimensions for type halfvec must be at least 1
LINE 1: SELECT '[1,2,3]'::halfvec(0);
^
SELECT '[1,2,3]'::halfvec(16001);
ERROR: dimensions for type halfvec cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::halfvec(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::halfvec(2)[];
ERROR: expected 2 dimensions, not 3

View File

@@ -1,5 +1,4 @@
SET enable_seqscan = off;
-- hamming
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
@@ -20,28 +19,7 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
(1 row)
DROP TABLE t;
-- jaccard
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
val
------
1111
1110
1100
0000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- varbit
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: type not supported for hnsw index

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
val
------
1111
1110
1100
0000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -1,102 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- L1
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,33 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 1);
ERROR: value 1 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
ERROR: value 101 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
ERROR: value 3 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m
SHOW hnsw.ef_search;
hnsw.ef_search
----------------
40
(1 row)
SET hnsw.ef_search = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
SET hnsw.ef_search = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
DROP TABLE t;

View File

@@ -1,112 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:4}/3
{1:1,2:2,3:3}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- L1
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l1_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <+> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- non-zero elements
CREATE TABLE t (val sparsevec(1001));
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
TRUNCATE t;
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:4}/3
{1:1,2:2,3:3}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l1_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <+> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,43 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----
(0 rows)
DROP TABLE t;
-- TODO move
CREATE TABLE t (val sparsevec(1001));
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
TRUNCATE t;
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
DROP TABLE t;

View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;

View File

@@ -1,142 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- L1
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- unlogged
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;
-- options
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 1);
ERROR: value 1 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
ERROR: value 101 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
ERROR: value 3 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m
SHOW hnsw.ef_search;
hnsw.ef_search
----------------
40
(1 row)
SET hnsw.ef_search = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
SET hnsw.ef_search = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,33 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -1,5 +1,4 @@
SET enable_seqscan = off;
-- hamming
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
@@ -20,7 +19,7 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
(1 row)
DROP TABLE t;
-- varbit
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
ERROR: type not supported for ivfflat index

View File

@@ -1,84 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,36 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -0,0 +1,14 @@
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
ERROR: value 0 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
SHOW ivfflat.probes;
ivfflat.probes
----------------
1
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;

View File

@@ -1,112 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- unlogged
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;
-- options
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
ERROR: value 0 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
SHOW ivfflat.probes;
ivfflat.probes
----------------
1
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,36 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -1,653 +0,0 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:-2,3:-4}/5'::sparsevec;
sparsevec
---------------
{1:-2,3:-4}/5
(1 row)
SELECT '{1:2.,3:4.}/5'::sparsevec;
sparsevec
-------------
{1:2,3:4}/5
(1 row)
SELECT ' { 1 : 1.5 , 3 : 3.5 } / 5 '::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:1.23456}/1'::sparsevec;
sparsevec
---------------
{1:1.23456}/1
(1 row)
SELECT '{1:hello,2:1}/2'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:hello,2:1}/2"
LINE 1: SELECT '{1:hello,2:1}/2'::sparsevec;
^
SELECT '{1:NaN,2:1}/2'::sparsevec;
ERROR: NaN not allowed in sparsevec
LINE 1: SELECT '{1:NaN,2:1}/2'::sparsevec;
^
SELECT '{1:Infinity,2:1}/2'::sparsevec;
ERROR: infinite value not allowed in sparsevec
LINE 1: SELECT '{1:Infinity,2:1}/2'::sparsevec;
^
SELECT '{1:-Infinity,2:1}/2'::sparsevec;
ERROR: infinite value not allowed in sparsevec
LINE 1: SELECT '{1:-Infinity,2:1}/2'::sparsevec;
^
SELECT '{1:1.5e38,2:-1.5e38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e+38,2:-1.5e+38}/2
(1 row)
SELECT '{1:1.5e+38,2:-1.5e+38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e+38,2:-1.5e+38}/2
(1 row)
SELECT '{1:1.5e-38,2:-1.5e-38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e-38,2:-1.5e-38}/2
(1 row)
SELECT '{1:4e38,2:1}/2'::sparsevec;
ERROR: "4e38" is out of range for type sparsevec
LINE 1: SELECT '{1:4e38,2:1}/2'::sparsevec;
^
SELECT '{1:-4e38,2:1}/2'::sparsevec;
ERROR: "-4e38" is out of range for type sparsevec
LINE 1: SELECT '{1:-4e38,2:1}/2'::sparsevec;
^
SELECT '{1:1e-46,2:1}/2'::sparsevec;
ERROR: "1e-46" is out of range for type sparsevec
LINE 1: SELECT '{1:1e-46,2:1}/2'::sparsevec;
^
SELECT '{1:-1e-46,2:1}/2'::sparsevec;
ERROR: "-1e-46" is out of range for type sparsevec
LINE 1: SELECT '{1:-1e-46,2:1}/2'::sparsevec;
^
SELECT ''::sparsevec;
ERROR: invalid input syntax for type sparsevec: ""
LINE 1: SELECT ''::sparsevec;
^
DETAIL: Vector contents must start with "{".
SELECT '{'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{"
LINE 1: SELECT '{'::sparsevec;
^
SELECT '{ '::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{ "
LINE 1: SELECT '{ '::sparsevec;
^
SELECT '{:'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:"
LINE 1: SELECT '{:'::sparsevec;
^
SELECT '{,'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{,"
LINE 1: SELECT '{,'::sparsevec;
^
SELECT '{}'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}"
LINE 1: SELECT '{}'::sparsevec;
^
DETAIL: Unexpected end of input.
SELECT '{}/'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}/"
LINE 1: SELECT '{}/'::sparsevec;
^
SELECT '{}/1'::sparsevec;
sparsevec
-----------
{}/1
(1 row)
SELECT '{}/1a'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}/1a"
LINE 1: SELECT '{}/1a'::sparsevec;
^
DETAIL: Junk after closing.
SELECT '{ }/1'::sparsevec;
sparsevec
-----------
{}/1
(1 row)
SELECT '{:}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:}/1"
LINE 1: SELECT '{:}/1'::sparsevec;
^
SELECT '{,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{,}/1"
LINE 1: SELECT '{,}/1'::sparsevec;
^
SELECT '{1,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1,}/1"
LINE 1: SELECT '{1,}/1'::sparsevec;
^
SELECT '{:1}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:1}/1"
LINE 1: SELECT '{:1}/1'::sparsevec;
^
SELECT '{1:}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:}/1"
LINE 1: SELECT '{1:}/1'::sparsevec;
^
SELECT '{1a:1}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1a:1}/1"
LINE 1: SELECT '{1a:1}/1'::sparsevec;
^
SELECT '{1:1a}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:1a}/1"
LINE 1: SELECT '{1:1a}/1'::sparsevec;
^
SELECT '{1:1,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:1,}/1"
LINE 1: SELECT '{1:1,}/1'::sparsevec;
^
SELECT '{1:0,2:1,3:0}/3'::sparsevec;
sparsevec
-----------
{2:1}/3
(1 row)
SELECT '{2:1,1:1}/2'::sparsevec;
sparsevec
-------------
{1:1,2:1}/2
(1 row)
SELECT '{1:1,1:1}/2'::sparsevec;
ERROR: sparsevec indices must not contain duplicates
LINE 1: SELECT '{1:1,1:1}/2'::sparsevec;
^
SELECT '{1:1,2:1,1:1}/2'::sparsevec;
ERROR: sparsevec indices must not contain duplicates
LINE 1: SELECT '{1:1,2:1,1:1}/2'::sparsevec;
^
SELECT '{}/5'::sparsevec;
sparsevec
-----------
{}/5
(1 row)
SELECT '{}/-1'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-1'::sparsevec;
^
SELECT '{}/1000000001'::sparsevec;
ERROR: sparsevec cannot have more than 1000000000 dimensions
LINE 1: SELECT '{}/1000000001'::sparsevec;
^
SELECT '{}/2147483648'::sparsevec;
ERROR: sparsevec cannot have more than 1000000000 dimensions
LINE 1: SELECT '{}/2147483648'::sparsevec;
^
SELECT '{}/-2147483649'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-2147483649'::sparsevec;
^
SELECT '{}/9223372036854775808'::sparsevec;
ERROR: sparsevec cannot have more than 1000000000 dimensions
LINE 1: SELECT '{}/9223372036854775808'::sparsevec;
^
SELECT '{}/-9223372036854775809'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-9223372036854775809'::sparsevec;
^
SELECT '{2147483647:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{2147483647:1}/1'::sparsevec;
^
SELECT '{2147483648:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{2147483648:1}/1'::sparsevec;
^
SELECT '{-2147483648:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{-2147483648:1}/1'::sparsevec;
^
SELECT '{-2147483649:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{-2147483649:1}/1'::sparsevec;
^
SELECT '{0:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{0:1}/1'::sparsevec;
^
SELECT '{2:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{2:1}/1'::sparsevec;
^
SELECT '{}/3'::sparsevec(3);
sparsevec
-----------
{}/3
(1 row)
SELECT '{}/3'::sparsevec(2);
ERROR: expected 2 dimensions, not 3
SELECT '{}/3'::sparsevec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '{}/3'::sparsevec(3, 2);
^
SELECT '{}/3'::sparsevec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '{}/3'::sparsevec('a');
^
SELECT '{}/3'::sparsevec(0);
ERROR: dimensions for type sparsevec must be at least 1
LINE 1: SELECT '{}/3'::sparsevec(0);
^
SELECT '{}/3'::sparsevec(1000000001);
ERROR: dimensions for type sparsevec cannot exceed 1000000000
LINE 1: SELECT '{}/3'::sparsevec(1000000001);
^
SELECT '{1:1,2:2,3:3}/3'::sparsevec < '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec < '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec <= '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec <= '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec = '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec = '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec != '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec != '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec >= '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec >= '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec > '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec > '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
0
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{}/3');
sparsevec_cmp
---------------
1
(1 row)
SELECT sparsevec_cmp('{}/3', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2}/2', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{1:1,2:2}/2');
sparsevec_cmp
---------------
1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2}/2', '{1:2,2:3,3:4}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:2,2:3}/2', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
1
(1 row)
SELECT round(l2_norm('{1:1,2:1}/2'::sparsevec)::numeric, 5);
round
---------
1.41421
(1 row)
SELECT l2_norm('{1:3,2:4}/2'::sparsevec);
l2_norm
---------
5
(1 row)
SELECT l2_norm('{2:1}/2'::sparsevec);
l2_norm
---------
1
(1 row)
SELECT l2_norm('{1:3e37,2:4e37}/2'::sparsevec)::real;
l2_norm
---------
5e+37
(1 row)
SELECT l2_norm('{}/2'::sparsevec);
l2_norm
---------
0
(1 row)
SELECT l2_norm('{1:2}/1'::sparsevec);
l2_norm
---------
2
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{1:3}/2'::sparsevec, '{2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{2:4}/2'::sparsevec, '{1:3}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{1:3,2:4}/2'::sparsevec, '{}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
l2_distance
-------------
1
(1 row)
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2';
?column?
----------
5
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
inner_product
---------------
10
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT inner_product('{1:1,3:3}/4'::sparsevec, '{2:2,4:4}/4');
inner_product
---------------
0
(1 row)
SELECT inner_product('{2:2,4:4}/4'::sparsevec, '{1:1,3:3}/4');
inner_product
---------------
0
(1 row)
SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
inner_product
---------------
18
(1 row)
SELECT inner_product('{1:1}/2'::sparsevec, '{}/2');
inner_product
---------------
0
(1 row)
SELECT inner_product('{}/2'::sparsevec, '{1:1}/2');
inner_product
---------------
0
(1 row)
SELECT inner_product('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
inner_product
---------------
Infinity
(1 row)
SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
inner_product
---------------
18
(1 row)
SELECT '{1:1,2:2}/2'::sparsevec <#> '{1:3,2:4}/2';
?column?
----------
-11
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1,2:1}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{2:2}/2'::sparsevec, '{1:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1.1,2:1.1}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1.1,2:-1.1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT '{1:1,2:2}/2'::sparsevec <=> '{1:2,2:4}/2';
?column?
----------
0
(1 row)
SELECT l1_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('{}/2'::sparsevec, '{2:1}/2');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT l1_distance('{1:3e38}/1'::sparsevec, '{1:-3e38}/1');
l1_distance
-------------
Infinity
(1 row)
SELECT l1_distance('{1:1,3:3,5:5,7:7}/8'::sparsevec, '{2:2,4:4,6:6,8:8}/8');
l1_distance
-------------
36
(1 row)
SELECT l1_distance('{1:1,3:3,5:5,7:7,9:9}/9'::sparsevec, '{2:2,4:4,6:6,8:8}/9');
l1_distance
-------------
45
(1 row)
SELECT '{}/2'::sparsevec <+> '{1:3,2:4}/2';
?column?
----------
7
(1 row)
SELECT l2_normalize('{1:3,2:4}/2'::sparsevec);
l2_normalize
-----------------
{1:0.6,2:0.8}/2
(1 row)
SELECT l2_normalize('{1:3}/2'::sparsevec);
l2_normalize
--------------
{1:1}/2
(1 row)
SELECT l2_normalize('{2:0.1}/2'::sparsevec);
l2_normalize
--------------
{2:1}/2
(1 row)
SELECT l2_normalize('{}/2'::sparsevec);
l2_normalize
--------------
{}/2
(1 row)
SELECT l2_normalize('{1:3e38}/1'::sparsevec);
l2_normalize
--------------
{1:1}/1
(1 row)
SELECT l2_normalize('{1:3e38,2:1e-37}/2'::sparsevec);
l2_normalize
--------------
{1:1}/2
(1 row)
SELECT l2_normalize('{2:3e37,4:3e-37,6:4e37,8:4e-37}/9'::sparsevec);
l2_normalize
-----------------
{2:0.6,6:0.8}/9
(1 row)

View File

@@ -0,0 +1,396 @@
SELECT '{1:1,2:2,3:3}/3'::sparsevec < '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec < '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec <= '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec <= '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec = '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec = '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec != '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec != '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec >= '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec >= '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec > '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec > '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
0
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{}/3');
sparsevec_cmp
---------------
1
(1 row)
SELECT sparsevec_cmp('{}/3', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2}/2', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{1:1,2:2}/2');
sparsevec_cmp
---------------
1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2}/2', '{1:2,2:3,3:4}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:2,2:3}/2', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
1
(1 row)
SELECT round(l2_norm('{1:1,2:1}/2'::sparsevec)::numeric, 5);
round
---------
1.41421
(1 row)
SELECT l2_norm('{1:3,2:4}/2'::sparsevec);
l2_norm
---------
5
(1 row)
SELECT l2_norm('{2:1}/2'::sparsevec);
l2_norm
---------
1
(1 row)
SELECT l2_norm('{1:3e37,2:4e37}/2'::sparsevec)::real;
l2_norm
---------
5e+37
(1 row)
SELECT l2_norm('{}/2'::sparsevec);
l2_norm
---------
0
(1 row)
SELECT l2_norm('{1:2}/1'::sparsevec);
l2_norm
---------
2
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{1:3}/2'::sparsevec, '{2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{2:4}/2'::sparsevec, '{1:3}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{1:3,2:4}/2'::sparsevec, '{}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
l2_distance
-------------
1
(1 row)
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2';
?column?
----------
5
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
inner_product
---------------
10
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT inner_product('{1:1,3:3}/4'::sparsevec, '{2:2,4:4}/4');
inner_product
---------------
0
(1 row)
SELECT inner_product('{2:2,4:4}/4'::sparsevec, '{1:1,3:3}/4');
inner_product
---------------
0
(1 row)
SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
inner_product
---------------
18
(1 row)
SELECT inner_product('{1:1}/2'::sparsevec, '{}/2');
inner_product
---------------
0
(1 row)
SELECT inner_product('{}/2'::sparsevec, '{1:1}/2');
inner_product
---------------
0
(1 row)
SELECT inner_product('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
inner_product
---------------
Infinity
(1 row)
SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
inner_product
---------------
18
(1 row)
SELECT '{1:1,2:2}/2'::sparsevec <#> '{1:3,2:4}/2';
?column?
----------
-11
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1,2:1}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{2:2}/2'::sparsevec, '{1:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1.1,2:1.1}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1.1,2:-1.1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT '{1:1,2:2}/2'::sparsevec <=> '{1:2,2:4}/2';
?column?
----------
0
(1 row)
SELECT l1_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('{}/2'::sparsevec, '{2:1}/2');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT l1_distance('{1:3e38}/1'::sparsevec, '{1:-3e38}/1');
l1_distance
-------------
Infinity
(1 row)
SELECT l1_distance('{1:1,3:3,5:5,7:7}/8'::sparsevec, '{2:2,4:4,6:6,8:8}/8');
l1_distance
-------------
36
(1 row)
SELECT l1_distance('{1:1,3:3,5:5,7:7,9:9}/9'::sparsevec, '{2:2,4:4,6:6,8:8}/9');
l1_distance
-------------
45
(1 row)
SELECT '{}/2'::sparsevec <+> '{1:3,2:4}/2';
?column?
----------
7
(1 row)
SELECT l2_normalize('{1:3,2:4}/2'::sparsevec);
l2_normalize
-----------------
{1:0.6,2:0.8}/2
(1 row)
SELECT l2_normalize('{1:3}/2'::sparsevec);
l2_normalize
--------------
{1:1}/2
(1 row)
SELECT l2_normalize('{2:0.1}/2'::sparsevec);
l2_normalize
--------------
{2:1}/2
(1 row)
SELECT l2_normalize('{}/2'::sparsevec);
l2_normalize
--------------
{}/2
(1 row)
SELECT l2_normalize('{1:3e38}/1'::sparsevec);
l2_normalize
--------------
{1:1}/1
(1 row)
SELECT l2_normalize('{1:3e38,2:1e-37}/2'::sparsevec);
l2_normalize
--------------
{1:1}/2
(1 row)
SELECT l2_normalize('{2:3e37,4:3e-37,6:4e37,8:4e-37}/9'::sparsevec);
l2_normalize
-----------------
{2:0.6,6:0.8}/9
(1 row)

View File

@@ -0,0 +1,225 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:-2,3:-4}/5'::sparsevec;
sparsevec
---------------
{1:-2,3:-4}/5
(1 row)
SELECT '{1:2.,3:4.}/5'::sparsevec;
sparsevec
-------------
{1:2,3:4}/5
(1 row)
SELECT ' { 1 : 1.5 , 3 : 3.5 } / 5 '::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:1.23456}/1'::sparsevec;
sparsevec
---------------
{1:1.23456}/1
(1 row)
SELECT '{1:hello,2:1}/2'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:hello,2:1}/2"
LINE 1: SELECT '{1:hello,2:1}/2'::sparsevec;
^
SELECT '{1:NaN,2:1}/2'::sparsevec;
ERROR: NaN not allowed in sparsevec
LINE 1: SELECT '{1:NaN,2:1}/2'::sparsevec;
^
SELECT '{1:Infinity,2:1}/2'::sparsevec;
ERROR: infinite value not allowed in sparsevec
LINE 1: SELECT '{1:Infinity,2:1}/2'::sparsevec;
^
SELECT '{1:-Infinity,2:1}/2'::sparsevec;
ERROR: infinite value not allowed in sparsevec
LINE 1: SELECT '{1:-Infinity,2:1}/2'::sparsevec;
^
SELECT '{1:1.5e38,2:-1.5e38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e+38,2:-1.5e+38}/2
(1 row)
SELECT '{1:1.5e+38,2:-1.5e+38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e+38,2:-1.5e+38}/2
(1 row)
SELECT '{1:1.5e-38,2:-1.5e-38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e-38,2:-1.5e-38}/2
(1 row)
SELECT '{1:4e38,2:1}/2'::sparsevec;
ERROR: "4e38" is out of range for type sparsevec
LINE 1: SELECT '{1:4e38,2:1}/2'::sparsevec;
^
SELECT '{1:-4e38,2:1}/2'::sparsevec;
ERROR: "-4e38" is out of range for type sparsevec
LINE 1: SELECT '{1:-4e38,2:1}/2'::sparsevec;
^
SELECT '{1:1e-46,2:1}/2'::sparsevec;
ERROR: "1e-46" is out of range for type sparsevec
LINE 1: SELECT '{1:1e-46,2:1}/2'::sparsevec;
^
SELECT '{1:-1e-46,2:1}/2'::sparsevec;
ERROR: "-1e-46" is out of range for type sparsevec
LINE 1: SELECT '{1:-1e-46,2:1}/2'::sparsevec;
^
SELECT ''::sparsevec;
ERROR: invalid input syntax for type sparsevec: ""
LINE 1: SELECT ''::sparsevec;
^
DETAIL: Vector contents must start with "{".
SELECT '{'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{"
LINE 1: SELECT '{'::sparsevec;
^
SELECT '{ '::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{ "
LINE 1: SELECT '{ '::sparsevec;
^
SELECT '{:'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:"
LINE 1: SELECT '{:'::sparsevec;
^
SELECT '{,'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{,"
LINE 1: SELECT '{,'::sparsevec;
^
SELECT '{}'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}"
LINE 1: SELECT '{}'::sparsevec;
^
DETAIL: Unexpected end of input.
SELECT '{}/'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}/"
LINE 1: SELECT '{}/'::sparsevec;
^
SELECT '{}/1'::sparsevec;
sparsevec
-----------
{}/1
(1 row)
SELECT '{}/1a'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}/1a"
LINE 1: SELECT '{}/1a'::sparsevec;
^
DETAIL: Junk after closing.
SELECT '{ }/1'::sparsevec;
sparsevec
-----------
{}/1
(1 row)
SELECT '{:}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:}/1"
LINE 1: SELECT '{:}/1'::sparsevec;
^
SELECT '{,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{,}/1"
LINE 1: SELECT '{,}/1'::sparsevec;
^
SELECT '{1,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1,}/1"
LINE 1: SELECT '{1,}/1'::sparsevec;
^
SELECT '{:1}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:1}/1"
LINE 1: SELECT '{:1}/1'::sparsevec;
^
SELECT '{1:}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:}/1"
LINE 1: SELECT '{1:}/1'::sparsevec;
^
SELECT '{1a:1}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1a:1}/1"
LINE 1: SELECT '{1a:1}/1'::sparsevec;
^
SELECT '{1:1a}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:1a}/1"
LINE 1: SELECT '{1:1a}/1'::sparsevec;
^
SELECT '{1:1,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:1,}/1"
LINE 1: SELECT '{1:1,}/1'::sparsevec;
^
SELECT '{1:0,2:1,3:0}/3'::sparsevec;
sparsevec
-----------
{2:1}/3
(1 row)
SELECT '{2:1,1:1}/2'::sparsevec;
sparsevec
-------------
{1:1,2:1}/2
(1 row)
SELECT '{1:1,1:1}/2'::sparsevec;
ERROR: indexes must not contain duplicates
LINE 1: SELECT '{1:1,1:1}/2'::sparsevec;
^
SELECT '{1:1,2:1,1:1}/2'::sparsevec;
ERROR: indexes must not contain duplicates
LINE 1: SELECT '{1:1,2:1,1:1}/2'::sparsevec;
^
SELECT '{}/5'::sparsevec;
sparsevec
-----------
{}/5
(1 row)
SELECT '{}/-1'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-1'::sparsevec;
^
SELECT '{}/1000001'::sparsevec;
ERROR: sparsevec cannot have more than 1000000 dimensions
LINE 1: SELECT '{}/1000001'::sparsevec;
^
SELECT '{0:1}/1'::sparsevec;
ERROR: index "0" is out of range for type sparsevec
LINE 1: SELECT '{0:1}/1'::sparsevec;
^
SELECT '{2:1}/1'::sparsevec;
ERROR: index must be less than or equal to dimensions
LINE 1: SELECT '{2:1}/1'::sparsevec;
^
SELECT '{}/3'::sparsevec(3);
sparsevec
-----------
{}/3
(1 row)
SELECT '{}/3'::sparsevec(2);
ERROR: expected 2 dimensions, not 3
SELECT '{}/3'::sparsevec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '{}/3'::sparsevec(3, 2);
^
SELECT '{}/3'::sparsevec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '{}/3'::sparsevec('a');
^
SELECT '{}/3'::sparsevec(0);
ERROR: dimensions for type sparsevec must be at least 1
LINE 1: SELECT '{}/3'::sparsevec(0);
^
SELECT '{}/3'::sparsevec(1000001);
ERROR: dimensions for type sparsevec cannot exceed 1000000
LINE 1: SELECT '{}/3'::sparsevec(1000001);
^

View File

@@ -1,179 +1,3 @@
SELECT '[1,2,3]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::vector;
vector
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "[hello,1]"
LINE 1: SELECT '[hello,1]'::vector;
^
SELECT '[NaN,1]'::vector;
ERROR: NaN not allowed in vector
LINE 1: SELECT '[NaN,1]'::vector;
^
SELECT '[Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[Infinity,1]'::vector;
^
SELECT '[-Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[-Infinity,1]'::vector;
^
SELECT '[1.5e38,-1.5e38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e+38,-1.5e+38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e-38,-1.5e-38]'::vector;
vector
--------------------
[1.5e-38,-1.5e-38]
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: "4e38" is out of range for type vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[-4e38,1]'::vector;
ERROR: "-4e38" is out of range for type vector
LINE 1: SELECT '[-4e38,1]'::vector;
^
SELECT '[1e-46,1]'::vector;
vector
--------
[0,1]
(1 row)
SELECT '[-1e-46,1]'::vector;
vector
--------
[-0,1]
(1 row)
SELECT '[1,2,3'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector;
^
SELECT '[1,2,3]9'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::vector;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::vector;
ERROR: invalid input syntax for type vector: "1,2,3"
LINE 1: SELECT '1,2,3'::vector;
^
DETAIL: Vector contents must start with "[".
SELECT ''::vector;
ERROR: invalid input syntax for type vector: ""
LINE 1: SELECT ''::vector;
^
DETAIL: Vector contents must start with "[".
SELECT '['::vector;
ERROR: invalid input syntax for type vector: "["
LINE 1: SELECT '['::vector;
^
SELECT '[ '::vector;
ERROR: invalid input syntax for type vector: "[ "
LINE 1: SELECT '[ '::vector;
^
SELECT '[,'::vector;
ERROR: invalid input syntax for type vector: "[,"
LINE 1: SELECT '[,'::vector;
^
SELECT '[]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[]'::vector;
^
SELECT '[ ]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[ ]'::vector;
^
SELECT '[,]'::vector;
ERROR: invalid input syntax for type vector: "[,]"
LINE 1: SELECT '[,]'::vector;
^
SELECT '[1,]'::vector;
ERROR: invalid input syntax for type vector: "[1,]"
LINE 1: SELECT '[1,]'::vector;
^
SELECT '[1a]'::vector;
ERROR: invalid input syntax for type vector: "[1a]"
LINE 1: SELECT '[1a]'::vector;
^
SELECT '[1,,3]'::vector;
ERROR: invalid input syntax for type vector: "[1,,3]"
LINE 1: SELECT '[1,,3]'::vector;
^
SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
^
SELECT '[1,2,3]'::vector('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::vector('a');
^
SELECT '[1,2,3]'::vector(0);
ERROR: dimensions for type vector must be at least 1
LINE 1: SELECT '[1,2,3]'::vector(0);
^
SELECT '[1,2,3]'::vector(16001);
ERROR: dimensions for type vector cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::vector(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::vector(2)[];
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector + '[4,5,6]';
?column?
----------
@@ -606,20 +430,6 @@ SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 2147483647, 10);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2147483647);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, -2147483644, 2147483647);
subvector
-----------
[1,2]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
avg
-----------

View File

@@ -0,0 +1,176 @@
SELECT '[1,2,3]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::vector;
vector
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "[hello,1]"
LINE 1: SELECT '[hello,1]'::vector;
^
SELECT '[NaN,1]'::vector;
ERROR: NaN not allowed in vector
LINE 1: SELECT '[NaN,1]'::vector;
^
SELECT '[Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[Infinity,1]'::vector;
^
SELECT '[-Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[-Infinity,1]'::vector;
^
SELECT '[1.5e38,-1.5e38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e+38,-1.5e+38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e-38,-1.5e-38]'::vector;
vector
--------------------
[1.5e-38,-1.5e-38]
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: "4e38" is out of range for type vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[-4e38,1]'::vector;
ERROR: "-4e38" is out of range for type vector
LINE 1: SELECT '[-4e38,1]'::vector;
^
SELECT '[1e-46,1]'::vector;
vector
--------
[0,1]
(1 row)
SELECT '[-1e-46,1]'::vector;
vector
--------
[-0,1]
(1 row)
SELECT '[1,2,3'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector;
^
SELECT '[1,2,3]9'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::vector;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::vector;
ERROR: invalid input syntax for type vector: "1,2,3"
LINE 1: SELECT '1,2,3'::vector;
^
DETAIL: Vector contents must start with "[".
SELECT ''::vector;
ERROR: invalid input syntax for type vector: ""
LINE 1: SELECT ''::vector;
^
DETAIL: Vector contents must start with "[".
SELECT '['::vector;
ERROR: invalid input syntax for type vector: "["
LINE 1: SELECT '['::vector;
^
SELECT '[ '::vector;
ERROR: invalid input syntax for type vector: "[ "
LINE 1: SELECT '[ '::vector;
^
SELECT '[,'::vector;
ERROR: invalid input syntax for type vector: "[,"
LINE 1: SELECT '[,'::vector;
^
SELECT '[]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[]'::vector;
^
SELECT '[ ]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[ ]'::vector;
^
SELECT '[,]'::vector;
ERROR: invalid input syntax for type vector: "[,]"
LINE 1: SELECT '[,]'::vector;
^
SELECT '[1,]'::vector;
ERROR: invalid input syntax for type vector: "[1,]"
LINE 1: SELECT '[1,]'::vector;
^
SELECT '[1a]'::vector;
ERROR: invalid input syntax for type vector: "[1a]"
LINE 1: SELECT '[1a]'::vector;
^
SELECT '[1,,3]'::vector;
ERROR: invalid input syntax for type vector: "[1,,3]"
LINE 1: SELECT '[1,,3]'::vector;
^
SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
^
SELECT '[1,2,3]'::vector('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::vector('a');
^
SELECT '[1,2,3]'::vector(0);
ERROR: dimensions for type vector must be at least 1
LINE 1: SELECT '[1,2,3]'::vector(0);
^
SELECT '[1,2,3]'::vector(16001);
ERROR: dimensions for type vector cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::vector(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::vector(2)[];
ERROR: expected 2 dimensions, not 3

View File

@@ -1,11 +0,0 @@
package PostgreSQL::Test::Cluster;
use PostgresNode;
sub new
{
my ($class, $name) = @_;
return get_new_node($name);
}
1;

View File

@@ -1,5 +0,0 @@
package PostgreSQL::Test::Utils;
use TestLib;
1;

View File

@@ -0,0 +1,8 @@
use PostgreSQL::Test::Cluster;
sub get_new_node
{
return PostgreSQL::Test::Cluster->new(@_);
}
1;

3
test/perl/TestLib.pm Normal file
View File

@@ -0,0 +1,3 @@
use PostgreSQL::Test::Utils;
1;

View File

@@ -1,34 +0,0 @@
SET enable_seqscan = off;
-- vector
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
SELECT * FROM t ORDER BY val;
DROP TABLE t;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
SELECT * FROM t ORDER BY val;
DROP TABLE t;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '{1:1,2:2,3:3}/3';
SELECT * FROM t ORDER BY val;
DROP TABLE t;

View File

@@ -0,0 +1,10 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
SELECT * FROM t ORDER BY val;
DROP TABLE t;

View File

@@ -0,0 +1,10 @@
SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '{1:1,2:2,3:3}/3';
SELECT * FROM t ORDER BY val;
DROP TABLE t;

10
test/sql/btree_vector.sql Normal file
View File

@@ -0,0 +1,10 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
SELECT * FROM t ORDER BY val;
DROP TABLE t;

View File

@@ -58,22 +58,6 @@ SELECT '{}/16001'::sparsevec::halfvec;
SELECT '{1:65520}/1'::sparsevec::halfvec;
SELECT '{1:1e-8}/1'::sparsevec::halfvec;
SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
SELECT '{NULL}'::real[]::sparsevec;
SELECT '{NaN}'::real[]::sparsevec;
SELECT '{Infinity}'::real[]::sparsevec;
SELECT '{-Infinity}'::real[]::sparsevec;
SELECT '{}'::real[]::sparsevec;
SELECT '{{1}}'::real[]::sparsevec;
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -1,42 +1,10 @@
-- vector
CREATE TABLE t (val vector(3), val2 halfvec(3), val3 sparsevec(3));
INSERT INTO t (val, val2, val3) VALUES ('[0,0,0]', '[0,0,0]', '{}/3'), ('[1,2,3]', '[1,2,3]', '{1:1,2:2,3:3}/3'), ('[1,1,1]', '[1,1,1]', '{1:1,2:1,3:1}/3'), (NULL, NULL, NULL);
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3), val2 halfvec(3), val3 sparsevec(3));
CREATE TABLE t2 (val vector(3));
\copy t TO 'results/vector.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/vector.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
DROP TABLE t;
DROP TABLE t2;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val halfvec(3));
\copy t TO 'results/halfvec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/halfvec.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
DROP TABLE t;
DROP TABLE t2;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE TABLE t2 (val sparsevec(3));
\copy t TO 'results/sparsevec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/sparsevec.bin' WITH (FORMAT binary)
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;

View File

@@ -1,42 +1,3 @@
SELECT '[1,2,3]'::halfvec;
SELECT '[-1,-2,-3]'::halfvec;
SELECT '[1.,2.,3.]'::halfvec;
SELECT ' [ 1, 2 , 3 ] '::halfvec;
SELECT '[1.23456]'::halfvec;
SELECT '[hello,1]'::halfvec;
SELECT '[NaN,1]'::halfvec;
SELECT '[Infinity,1]'::halfvec;
SELECT '[-Infinity,1]'::halfvec;
SELECT '[65519,-65519]'::halfvec;
SELECT '[65520,-65520]'::halfvec;
SELECT '[1e-8,-1e-8]'::halfvec;
SELECT '[4e38,1]'::halfvec;
SELECT '[1e-46,1]'::halfvec;
SELECT '[1,2,3'::halfvec;
SELECT '[1,2,3]9'::halfvec;
SELECT '1,2,3'::halfvec;
SELECT ''::halfvec;
SELECT '['::halfvec;
SELECT '[ '::halfvec;
SELECT '[,'::halfvec;
SELECT '[]'::halfvec;
SELECT '[ ]'::halfvec;
SELECT '[,]'::halfvec;
SELECT '[1,]'::halfvec;
SELECT '[1a]'::halfvec;
SELECT '[1,,3]'::halfvec;
SELECT '[1, ,3]'::halfvec;
SELECT '[1,2,3]'::halfvec(3);
SELECT '[1,2,3]'::halfvec(2);
SELECT '[1,2,3]'::halfvec(3, 2);
SELECT '[1,2,3]'::halfvec('a');
SELECT '[1,2,3]'::halfvec(0);
SELECT '[1,2,3]'::halfvec(16001);
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
SELECT '{"[1,2,3]"}'::halfvec(2)[];
SELECT '[1,2,3]'::halfvec + '[4,5,6]';
SELECT '[65519]'::halfvec + '[65519]';
SELECT '[1,2]'::halfvec + '[3]';
@@ -129,9 +90,6 @@ SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
SELECT subvector('[1,2,3,4,5]'::halfvec, 2147483647, 10);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2147483647);
SELECT subvector('[1,2,3,4,5]'::halfvec, -2147483644, 2147483647);
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]', NULL]) v;

View File

@@ -0,0 +1,38 @@
SELECT '[1,2,3]'::halfvec;
SELECT '[-1,-2,-3]'::halfvec;
SELECT '[1.,2.,3.]'::halfvec;
SELECT ' [ 1, 2 , 3 ] '::halfvec;
SELECT '[1.23456]'::halfvec;
SELECT '[hello,1]'::halfvec;
SELECT '[NaN,1]'::halfvec;
SELECT '[Infinity,1]'::halfvec;
SELECT '[-Infinity,1]'::halfvec;
SELECT '[65519,-65519]'::halfvec;
SELECT '[65520,-65520]'::halfvec;
SELECT '[1e-8,-1e-8]'::halfvec;
SELECT '[4e38,1]'::halfvec;
SELECT '[1e-46,1]'::halfvec;
SELECT '[1,2,3'::halfvec;
SELECT '[1,2,3]9'::halfvec;
SELECT '1,2,3'::halfvec;
SELECT ''::halfvec;
SELECT '['::halfvec;
SELECT '[ '::halfvec;
SELECT '[,'::halfvec;
SELECT '[]'::halfvec;
SELECT '[ ]'::halfvec;
SELECT '[,]'::halfvec;
SELECT '[1,]'::halfvec;
SELECT '[1a]'::halfvec;
SELECT '[1,,3]'::halfvec;
SELECT '[1, ,3]'::halfvec;
SELECT '[1,2,3]'::halfvec(3);
SELECT '[1,2,3]'::halfvec(2);
SELECT '[1,2,3]'::halfvec(3, 2);
SELECT '[1,2,3]'::halfvec('a');
SELECT '[1,2,3]'::halfvec(0);
SELECT '[1,2,3]'::halfvec(16001);
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
SELECT '{"[1,2,3]"}'::halfvec(2)[];

View File

@@ -1,7 +1,5 @@
SET enable_seqscan = off;
-- hamming
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
@@ -13,21 +11,7 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
DROP TABLE t;
-- jaccard
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
DROP TABLE t;
-- varbit
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);

View File

@@ -0,0 +1,12 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
DROP TABLE t;

View File

@@ -1,58 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
DROP TABLE t;
-- inner product
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
DROP TABLE t;
-- cosine
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t;
-- L1
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
DROP TABLE t;

View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t;

Some files were not shown because too many files have changed in this diff Show More