mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 20:15:46 +08:00
Compare commits
24 Commits
sparsevec
...
sparsevec-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5425dd2357 | ||
|
|
7ca7a64dbb | ||
|
|
aaa2d644ce | ||
|
|
253acbccf4 | ||
|
|
060d299e4b | ||
|
|
d7354a86a8 | ||
|
|
daba71694b | ||
|
|
506dd2b44a | ||
|
|
bcf41f5f66 | ||
|
|
abac7a3f77 | ||
|
|
32a502c838 | ||
|
|
1134e52762 | ||
|
|
3ef632e042 | ||
|
|
e2a527ffda | ||
|
|
835f010257 | ||
|
|
d6044dd423 | ||
|
|
c75634a03c | ||
|
|
ab7b2ed39e | ||
|
|
499b6bc2c9 | ||
|
|
741c6a8a7b | ||
|
|
1c82bdd932 | ||
|
|
94a444f029 | ||
|
|
7ee9074a9c | ||
|
|
2f2f3631a8 |
26
.github/workflows/build.yml
vendored
26
.github/workflows/build.yml
vendored
@@ -40,13 +40,21 @@ jobs:
|
||||
sudo apt-get install libipc-run-perl
|
||||
- run: make prove_installcheck
|
||||
mac:
|
||||
runs-on: macos-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
if: ${{ !startsWith(github.ref_name, 'windows') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- postgres: 16
|
||||
os: macos-14
|
||||
- postgres: 14
|
||||
os: macos-12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres@v1
|
||||
with:
|
||||
postgres-version: 14
|
||||
postgres-version: ${{ matrix.postgres }}
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
|
||||
@@ -54,13 +62,19 @@ jobs:
|
||||
- run: make installcheck
|
||||
- if: ${{ failure() }}
|
||||
run: cat regression.diffs
|
||||
# Homebrew Postgres does not enable TAP tests, so need to download
|
||||
- run: |
|
||||
brew install cpanm
|
||||
cpanm --notest IPC::Run
|
||||
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
|
||||
tar xf REL_14_10.tar.gz
|
||||
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
|
||||
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
|
||||
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz
|
||||
tar xf $TAG.tar.gz
|
||||
mv postgres-$TAG postgres
|
||||
env:
|
||||
TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
|
||||
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
|
||||
env:
|
||||
PERL5LIB: /Users/runner/perl5/lib/perl5
|
||||
- run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
|
||||
env:
|
||||
PG_CFLAGS: -DUSE_ASSERT_CHECKING
|
||||
windows:
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
## 0.7.0 (unreleased)
|
||||
|
||||
- Added `halfvec` type
|
||||
- Added `sparsevec` type
|
||||
- Added support for `bit` vectors to HNSW
|
||||
- Added `hamming_distance` function
|
||||
- Added `jaccard_distance` function
|
||||
- Added `quantize_binary` function
|
||||
- Added `subvector` function
|
||||
- Updated comparison operators to support vectors with different dimensions
|
||||
|
||||
## 0.6.2 (2024-03-18)
|
||||
|
||||
|
||||
6
Makefile
6
Makefile
@@ -3,8 +3,8 @@ EXTVERSION = 0.6.2
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
OBJS = src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
|
||||
HEADERS = src/sparsevec.h src/vector.h
|
||||
OBJS = src/bitvector.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
|
||||
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
|
||||
|
||||
TESTS = $(wildcard test/sql/*.sql)
|
||||
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
|
||||
@@ -12,7 +12,7 @@ REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
|
||||
|
||||
OPTFLAGS = -march=native
|
||||
|
||||
# Mac ARM doesn't support -march=native
|
||||
# Mac ARM doesn't always support -march=native
|
||||
ifeq ($(shell uname -s), Darwin)
|
||||
ifeq ($(shell uname -p), arm)
|
||||
# no difference with -march=armv8.5-a
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.6.2
|
||||
|
||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
HEADERS = src\sparsevec.h src\vector.h
|
||||
OBJS = src\bitvector.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
|
||||
|
||||
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
|
||||
REGRESS = bit_functions btree cast copy halfvec_functions halfvec_input hnsw_bit_hamming hnsw_bit_jaccard hnsw_halfvec_cosine hnsw_halfvec_ip hnsw_halfvec_l2 hnsw_options hnsw_sparsevec_cosine hnsw_sparsevec_ip hnsw_sparsevec_l2 hnsw_unlogged hnsw_vector_cosine hnsw_vector_ip hnsw_vector_l2 ivfflat_options ivfflat_unlogged ivfflat_vector_cosine ivfflat_vector_ip ivfflat_vector_l2 sparsevec_functions sparsevec_input vector_functions vector_input
|
||||
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
|
||||
|
||||
# For /arch flags
|
||||
|
||||
112
README.md
112
README.md
@@ -221,7 +221,24 @@ Cosine distance
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
Hamming distance - unreleased
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
|
||||
```
|
||||
|
||||
Jaccard distance - unreleased
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
|
||||
```
|
||||
|
||||
Supported types are:
|
||||
|
||||
- `vector` - up to 2,000 dimensions
|
||||
- `halfvec` - up to 4,000 dimensions (unreleased)
|
||||
- `bit` - up to 64,000 dimensions (unreleased)
|
||||
- `sparsevec` - up to 1,000 non-zero elements (unreleased)
|
||||
|
||||
### Index Options
|
||||
|
||||
@@ -326,7 +343,9 @@ Cosine distance
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
Supported types are:
|
||||
|
||||
- `vector` - up to 2,000 dimensions
|
||||
|
||||
### Query Options
|
||||
|
||||
@@ -523,26 +542,6 @@ Scale vertically by increasing memory, CPU, and storage on a single instance. Us
|
||||
|
||||
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
|
||||
|
||||
## Sparse Vectors
|
||||
|
||||
Create a sparse vector column with 10 dimensions
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (id bigserial PRIMARY KEY, embedding sparsevec(10));
|
||||
```
|
||||
|
||||
Insert vectors
|
||||
|
||||
```sql
|
||||
INSERT INTO items (embedding) VALUES ('{0:1,1:2,2:3}/10'), ('{0:4,1:5,2:6}/10');
|
||||
```
|
||||
|
||||
Get the nearest neighbors by L2 distance
|
||||
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <-> '{0:3,1:1,2:2}/10' LIMIT 5;
|
||||
```
|
||||
|
||||
## Languages
|
||||
|
||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||
@@ -719,6 +718,11 @@ Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosi
|
||||
|
||||
## Reference
|
||||
|
||||
- [Vector](#vector-type)
|
||||
- [Halfvec](#halfvec-type)
|
||||
- [Bit](#bit-type)
|
||||
- [Sparsevec](#sparsevec-type)
|
||||
|
||||
### Vector Type
|
||||
|
||||
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single-precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
|
||||
@@ -742,6 +746,8 @@ cosine_distance(vector, vector) → double precision | cosine distance |
|
||||
inner_product(vector, vector) → double precision | inner product |
|
||||
l2_distance(vector, vector) → double precision | Euclidean distance |
|
||||
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
|
||||
quantize_binary(vector) → bit | quantize | unreleased
|
||||
subvector(vector, integer, integer) → vector | subvector | unreleased
|
||||
vector_dims(vector) → integer | number of dimensions |
|
||||
vector_norm(vector) → double precision | Euclidean norm |
|
||||
|
||||
@@ -752,6 +758,68 @@ Function | Description | Added
|
||||
avg(vector) → vector | average |
|
||||
sum(vector) → vector | sum | 0.5.0
|
||||
|
||||
### Halfvec Type
|
||||
|
||||
Each half vector takes `2 * dimensions + 8` bytes of storage. Each element is a half-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Half vectors can have up to 16,000 dimensions.
|
||||
|
||||
### Halfvec Operators
|
||||
|
||||
Operator | Description | Added
|
||||
--- | --- | ---
|
||||
<-> | Euclidean distance | unreleased
|
||||
<#> | negative inner product | unreleased
|
||||
<=> | cosine distance | unreleased
|
||||
|
||||
### Halfvec Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
cosine_distance(halfvec, halfvec) → double precision | cosine distance | unreleased
|
||||
inner_product(halfvec, halfvec) → double precision | inner product | unreleased
|
||||
l2_distance(halfvec, halfvec) → double precision | Euclidean distance | unreleased
|
||||
l1_distance(halfvec, halfvec) → double precision | taxicab distance | unreleased
|
||||
quantize_binary(halfvec) → bit | quantize | unreleased
|
||||
subvector(halfvec, integer, integer) → halfvec | subvector | unreleased
|
||||
|
||||
### Bit Type
|
||||
|
||||
Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres docs](https://www.postgresql.org/docs/current/datatype-bit.html) for more info.
|
||||
|
||||
### Bit Operators
|
||||
|
||||
Operator | Description | Added
|
||||
--- | --- | ---
|
||||
<~> | Hamming distance | unreleased
|
||||
<%> | Jaccard distance | unreleased
|
||||
|
||||
### Bit Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
hamming_distance(bit, bit) → double precision | Hamming distance | unreleased
|
||||
jaccard_distance(bit, bit) → double precision | Jaccard distance | unreleased
|
||||
|
||||
### Sparsevec Type
|
||||
|
||||
Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each element is a single-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Sparse vectors can have up to 100,000 dimensions.
|
||||
|
||||
### Sparsevec Operators
|
||||
|
||||
Operator | Description | Added
|
||||
--- | --- | ---
|
||||
<-> | Euclidean distance | unreleased
|
||||
<#> | negative inner product | unreleased
|
||||
<=> | cosine distance | unreleased
|
||||
|
||||
### Sparsevec Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | unreleased
|
||||
inner_product(sparsevec, sparsevec) → double precision | inner product | unreleased
|
||||
l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | unreleased
|
||||
l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | unreleased
|
||||
|
||||
## Installation Notes - Linux and Mac
|
||||
|
||||
### Postgres Location
|
||||
|
||||
@@ -1,6 +1,170 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.0'" to load this file. \quit
|
||||
|
||||
CREATE FUNCTION quantize_binary(vector) RETURNS bit
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <~> (
|
||||
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
|
||||
COMMUTATOR = '<~>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <%> (
|
||||
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
|
||||
COMMUTATOR = '<%>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit);
|
||||
|
||||
CREATE TYPE halfvec;
|
||||
|
||||
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_out(halfvec) RETURNS cstring
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_typmod_in(cstring[]) RETURNS integer
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_recv(internal, oid, integer) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_send(halfvec) RETURNS bytea
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE TYPE halfvec (
|
||||
INPUT = halfvec_in,
|
||||
OUTPUT = halfvec_out,
|
||||
TYPMOD_IN = halfvec_typmod_in,
|
||||
RECEIVE = halfvec_recv,
|
||||
SEND = halfvec_send,
|
||||
STORAGE = external
|
||||
);
|
||||
|
||||
CREATE FUNCTION l2_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION inner_product(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION cosine_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION l1_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_norm(halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION quantize_binary(halfvec) RETURNS bit
|
||||
AS 'MODULE_PATHNAME', 'halfvec_quantize_binary' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(halfvec, int, int) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME', 'halfvec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_l2_squared_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_negative_inner_product(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec(halfvec, integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(integer[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(real[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(double precision[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(numeric[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_to_float4(halfvec, integer, boolean) RETURNS real[]
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE CAST (halfvec AS halfvec)
|
||||
WITH FUNCTION halfvec(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (halfvec AS real[])
|
||||
WITH FUNCTION halfvec_to_float4(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (integer[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(integer[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (real[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(real[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (double precision[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(double precision[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (numeric[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(numeric[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <#> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_negative_inner_product,
|
||||
COMMUTATOR = '<#>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <=> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = cosine_distance,
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_l2_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_ip_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 2 halfvec_norm(halfvec);
|
||||
|
||||
CREATE FUNCTION halfvec_to_vector(halfvec, integer, boolean) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_to_halfvec(vector, integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE CAST (halfvec AS vector)
|
||||
WITH FUNCTION halfvec_to_vector(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (vector AS halfvec)
|
||||
WITH FUNCTION vector_to_halfvec(vector, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE TYPE sparsevec;
|
||||
|
||||
CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
|
||||
@@ -39,6 +203,9 @@ CREATE FUNCTION cosine_distance(sparsevec, sparsevec) RETURNS float8
|
||||
CREATE FUNCTION sparsevec_norm(sparsevec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(sparsevec, int, int) RETURNS sparsevec
|
||||
AS 'MODULE_PATHNAME', 'sparsevec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION sparsevec_l2_squared_distance(sparsevec, sparsevec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
|
||||
185
sql/vector.sql
185
sql/vector.sql
@@ -58,6 +58,12 @@ CREATE FUNCTION vector_sub(vector, vector) RETURNS vector
|
||||
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION quantize_binary(vector) RETURNS bit
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- vector private functions
|
||||
|
||||
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
|
||||
@@ -288,6 +294,182 @@ CREATE OPERATOR CLASS vector_cosine_ops
|
||||
FUNCTION 1 vector_negative_inner_product(vector, vector),
|
||||
FUNCTION 2 vector_norm(vector);
|
||||
|
||||
-- bit functions
|
||||
|
||||
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <~> (
|
||||
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
|
||||
COMMUTATOR = '<~>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <%> (
|
||||
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
|
||||
COMMUTATOR = '<%>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit);
|
||||
|
||||
-- halfvec type
|
||||
|
||||
CREATE TYPE halfvec;
|
||||
|
||||
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_out(halfvec) RETURNS cstring
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_typmod_in(cstring[]) RETURNS integer
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_recv(internal, oid, integer) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_send(halfvec) RETURNS bytea
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE TYPE halfvec (
|
||||
INPUT = halfvec_in,
|
||||
OUTPUT = halfvec_out,
|
||||
TYPMOD_IN = halfvec_typmod_in,
|
||||
RECEIVE = halfvec_recv,
|
||||
SEND = halfvec_send,
|
||||
STORAGE = external
|
||||
);
|
||||
|
||||
-- halfvec functions
|
||||
|
||||
CREATE FUNCTION l2_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION inner_product(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION cosine_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION l1_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'halfvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_norm(halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION quantize_binary(halfvec) RETURNS bit
|
||||
AS 'MODULE_PATHNAME', 'halfvec_quantize_binary' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(halfvec, int, int) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME', 'halfvec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- halfvec private functions
|
||||
|
||||
CREATE FUNCTION halfvec_l2_squared_distance(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_negative_inner_product(halfvec, halfvec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- halfvec cast functions
|
||||
|
||||
CREATE FUNCTION halfvec(halfvec, integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(integer[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(real[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(double precision[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_halfvec(numeric[], integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_to_float4(halfvec, integer, boolean) RETURNS real[]
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- halfvec casts
|
||||
|
||||
CREATE CAST (halfvec AS halfvec)
|
||||
WITH FUNCTION halfvec(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (halfvec AS real[])
|
||||
WITH FUNCTION halfvec_to_float4(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (integer[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(integer[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (real[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(real[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (double precision[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(double precision[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (numeric[] AS halfvec)
|
||||
WITH FUNCTION array_to_halfvec(numeric[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
-- halfvec operators
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <#> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_negative_inner_product,
|
||||
COMMUTATOR = '<#>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <=> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = cosine_distance,
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
-- halfvec opclasses
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_l2_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_ip_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 2 halfvec_norm(halfvec);
|
||||
|
||||
-- extension casts
|
||||
|
||||
CREATE FUNCTION halfvec_to_vector(halfvec, integer, boolean) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_to_halfvec(vector, integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE CAST (halfvec AS vector)
|
||||
WITH FUNCTION halfvec_to_vector(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (vector AS halfvec)
|
||||
WITH FUNCTION vector_to_halfvec(vector, integer, boolean) AS IMPLICIT;
|
||||
|
||||
--- sparsevec type
|
||||
|
||||
CREATE TYPE sparsevec;
|
||||
@@ -330,6 +512,9 @@ CREATE FUNCTION cosine_distance(sparsevec, sparsevec) RETURNS float8
|
||||
CREATE FUNCTION sparsevec_norm(sparsevec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(sparsevec, int, int) RETURNS sparsevec
|
||||
AS 'MODULE_PATHNAME', 'sparsevec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- sparsevec private functions
|
||||
|
||||
CREATE FUNCTION sparsevec_l2_squared_distance(sparsevec, sparsevec) RETURNS float8
|
||||
|
||||
90
src/bitvector.c
Normal file
90
src/bitvector.c
Normal file
@@ -0,0 +1,90 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include "bitvector.h"
|
||||
#include "port/pg_bitutils.h"
|
||||
#include "utils/varbit.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
#include "varatt.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Allocate and initialize a new bit vector
|
||||
*/
|
||||
VarBit *
|
||||
InitBitVector(int dim)
|
||||
{
|
||||
VarBit *result;
|
||||
int size;
|
||||
|
||||
size = VARBITTOTALLEN(dim);
|
||||
result = (VarBit *) palloc0(size);
|
||||
SET_VARSIZE(result, size);
|
||||
VARBITLEN(result) = dim;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure same dimensions
|
||||
*/
|
||||
static inline void
|
||||
CheckDims(VarBit *a, VarBit *b)
|
||||
{
|
||||
if (VARBITLEN(a) != VARBITLEN(b))
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_DATA_EXCEPTION),
|
||||
errmsg("different bit lengths %u and %u", VARBITLEN(a), VARBITLEN(b))));
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the Hamming distance between two bit vectors
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
|
||||
Datum
|
||||
hamming_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
VarBit *a = PG_GETARG_VARBIT_P(0);
|
||||
VarBit *b = PG_GETARG_VARBIT_P(1);
|
||||
unsigned char *ax = VARBITS(a);
|
||||
unsigned char *bx = VARBITS(b);
|
||||
uint64 distance = 0;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* TODO Improve performance */
|
||||
for (uint32 i = 0; i < VARBITBYTES(a); i++)
|
||||
distance += pg_number_of_ones[ax[i] ^ bx[i]];
|
||||
|
||||
PG_RETURN_FLOAT8((double) distance);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the Jaccard distance between two bit vectors
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(jaccard_distance);
|
||||
Datum
|
||||
jaccard_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
VarBit *a = PG_GETARG_VARBIT_P(0);
|
||||
VarBit *b = PG_GETARG_VARBIT_P(1);
|
||||
unsigned char *ax = VARBITS(a);
|
||||
unsigned char *bx = VARBITS(b);
|
||||
uint64 ab = 0;
|
||||
uint64 aa;
|
||||
uint64 bb;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
/* TODO Improve performance */
|
||||
for (uint32 i = 0; i < VARBITBYTES(a); i++)
|
||||
ab += pg_number_of_ones[ax[i] & bx[i]];
|
||||
|
||||
if (ab == 0)
|
||||
PG_RETURN_FLOAT8(1);
|
||||
|
||||
aa = pg_popcount((char *) ax, VARBITBYTES(a));
|
||||
bb = pg_popcount((char *) bx, VARBITBYTES(b));
|
||||
|
||||
PG_RETURN_FLOAT8(1 - (ab / ((double) (aa + bb - ab))));
|
||||
}
|
||||
8
src/bitvector.h
Normal file
8
src/bitvector.h
Normal file
@@ -0,0 +1,8 @@
|
||||
#ifndef BITVECTOR_H
|
||||
#define BITVECTOR_H
|
||||
|
||||
#include "utils/varbit.h"
|
||||
|
||||
VarBit *InitBitVector(int dim);
|
||||
|
||||
#endif
|
||||
1020
src/halfvec.c
Normal file
1020
src/halfvec.c
Normal file
File diff suppressed because it is too large
Load Diff
43
src/halfvec.h
Normal file
43
src/halfvec.h
Normal file
@@ -0,0 +1,43 @@
|
||||
#ifndef HALFVEC_H
|
||||
#define HALFVEC_H
|
||||
|
||||
#define __STDC_WANT_IEC_60559_TYPES_EXT__
|
||||
|
||||
#include <float.h>
|
||||
|
||||
#include "vector.h"
|
||||
|
||||
#ifdef __FLT16_MAX__
|
||||
#define FLT16_SUPPORT
|
||||
#endif
|
||||
|
||||
#ifdef FLT16_SUPPORT
|
||||
#define half _Float16
|
||||
#define HALF_MAX FLT16_MAX
|
||||
#else
|
||||
/* TODO #pragma message("")? */
|
||||
#define half uint16
|
||||
#define HALF_MAX 65504
|
||||
#endif
|
||||
|
||||
#define HALFVEC_MAX_DIM VECTOR_MAX_DIM
|
||||
|
||||
#define HALFVEC_SIZE(_dim) (offsetof(HalfVector, x) + sizeof(half)*(_dim))
|
||||
#define DatumGetHalfVector(x) ((HalfVector *) PG_DETOAST_DATUM(x))
|
||||
#define PG_GETARG_HALFVEC_P(x) DatumGetHalfVector(PG_GETARG_DATUM(x))
|
||||
#define PG_RETURN_HALFVEC_P(x) PG_RETURN_POINTER(x)
|
||||
|
||||
typedef struct HalfVector
|
||||
{
|
||||
int32 vl_len_; /* varlena header (do not touch directly!) */
|
||||
int16 dim; /* number of dimensions */
|
||||
int16 unused;
|
||||
half x[FLEXIBLE_ARRAY_MEMBER];
|
||||
} HalfVector;
|
||||
|
||||
HalfVector *InitHalfVector(int dim);
|
||||
float HalfToFloat4(half num);
|
||||
half Float4ToHalf(float num);
|
||||
half Float4ToHalfUnchecked(float num);
|
||||
|
||||
#endif
|
||||
@@ -59,6 +59,8 @@
|
||||
typedef enum HnswType
|
||||
{
|
||||
HNSW_TYPE_VECTOR,
|
||||
HNSW_TYPE_HALFVEC,
|
||||
HNSW_TYPE_BIT,
|
||||
HNSW_TYPE_SPARSEVEC
|
||||
} HnswType;
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
#include "access/xact.h"
|
||||
#include "access/xloginsert.h"
|
||||
#include "catalog/index.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
#include "commands/progress.h"
|
||||
#include "hnsw.h"
|
||||
#include "miscadmin.h"
|
||||
@@ -668,13 +669,31 @@ HnswSharedMemoryAlloc(Size size, void *state)
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get max dimensions
|
||||
*/
|
||||
static int
|
||||
GetMaxDimensions(HnswType type)
|
||||
{
|
||||
int maxDimensions = HNSW_MAX_DIM;
|
||||
|
||||
if (type == HNSW_TYPE_HALFVEC)
|
||||
maxDimensions *= 2;
|
||||
else if (type == HNSW_TYPE_BIT)
|
||||
maxDimensions *= 32;
|
||||
else if (type == HNSW_TYPE_SPARSEVEC)
|
||||
maxDimensions = INT_MAX;
|
||||
|
||||
return maxDimensions;
|
||||
}
|
||||
|
||||
/*
|
||||
* Initialize the build state
|
||||
*/
|
||||
static void
|
||||
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
|
||||
{
|
||||
int maxDimensions = HNSW_MAX_DIM;
|
||||
int maxDimensions;
|
||||
|
||||
buildstate->heap = heap;
|
||||
buildstate->index = index;
|
||||
@@ -686,9 +705,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
||||
buildstate->efConstruction = HnswGetEfConstruction(index);
|
||||
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
|
||||
|
||||
/* No limit on sparse vector dimensions */
|
||||
if (buildstate->type == HNSW_TYPE_SPARSEVEC)
|
||||
maxDimensions = INT_MAX;
|
||||
maxDimensions = GetMaxDimensions(buildstate->type);
|
||||
|
||||
/* Require column to have dimensions to be indexed */
|
||||
if (buildstate->dimensions < 0)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
|
||||
#include "access/generic_xlog.h"
|
||||
#include "catalog/pg_type.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
#include "halfvec.h"
|
||||
#include "hnsw.h"
|
||||
#include "lib/pairingheap.h"
|
||||
#include "sparsevec.h"
|
||||
@@ -153,25 +155,32 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
|
||||
}
|
||||
|
||||
/*
|
||||
* Get vector type
|
||||
* Get type
|
||||
*/
|
||||
HnswType
|
||||
HnswGetType(Relation index)
|
||||
{
|
||||
Oid typeOid = TupleDescAttr(index->rd_att, 0)->atttypid;
|
||||
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
|
||||
HeapTuple tuple;
|
||||
Form_pg_type type;
|
||||
int result;
|
||||
|
||||
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typeOid));
|
||||
if (typid == BITOID || typid == VARBITOID)
|
||||
return HNSW_TYPE_BIT;
|
||||
|
||||
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid));
|
||||
if (!HeapTupleIsValid(tuple))
|
||||
elog(ERROR, "cache lookup failed for type %u", typeOid);
|
||||
elog(ERROR, "cache lookup failed for type %u", typid);
|
||||
|
||||
type = (Form_pg_type) GETSTRUCT(tuple);
|
||||
if (strcmp(NameStr(type->typname), "sparsevec") == 0)
|
||||
if (strcmp(NameStr(type->typname), "vector") == 0)
|
||||
result = HNSW_TYPE_VECTOR;
|
||||
else if (strcmp(NameStr(type->typname), "halfvec") == 0)
|
||||
result = HNSW_TYPE_HALFVEC;
|
||||
else if (strcmp(NameStr(type->typname), "sparsevec") == 0)
|
||||
result = HNSW_TYPE_SPARSEVEC;
|
||||
else
|
||||
result = HNSW_TYPE_VECTOR;
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
ReleaseSysCache(tuple);
|
||||
|
||||
@@ -204,6 +213,19 @@ HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type)
|
||||
|
||||
*value = PointerGetDatum(result);
|
||||
}
|
||||
else if (type == HNSW_TYPE_HALFVEC)
|
||||
{
|
||||
HalfVector *v = DatumGetHalfVector(*value);
|
||||
HalfVector *result = InitHalfVector(v->dim);
|
||||
|
||||
for (int i = 0; i < v->dim; i++)
|
||||
{
|
||||
/* TODO Fix */
|
||||
result->x[i] = Float4ToHalfUnchecked(HalfToFloat4(v->x[i]) / norm);
|
||||
}
|
||||
|
||||
*value = PointerGetDatum(result);
|
||||
}
|
||||
else if (type == HNSW_TYPE_SPARSEVEC)
|
||||
{
|
||||
SparseVector *v = DatumGetSparseVector(*value);
|
||||
|
||||
@@ -404,7 +404,6 @@ sparsevec_out(PG_FUNCTION_ARGS)
|
||||
AppendChar(ptr, '}');
|
||||
AppendChar(ptr, '/');
|
||||
AppendInt(ptr, sparsevec->dim);
|
||||
|
||||
*ptr = '\0';
|
||||
|
||||
PG_FREE_IF_COPY(sparsevec, 0);
|
||||
@@ -777,3 +776,55 @@ sparsevec_norm(PG_FUNCTION_ARGS)
|
||||
|
||||
PG_RETURN_FLOAT8(sqrt(norm));
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a subvector
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_subvector);
|
||||
Datum
|
||||
sparsevec_subvector(PG_FUNCTION_ARGS)
|
||||
{
|
||||
SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
|
||||
int32 start = PG_GETARG_INT32(1);
|
||||
int32 count = PG_GETARG_INT32(2);
|
||||
int32 end = start + count;
|
||||
float *ax = SPARSEVEC_VALUES(a);
|
||||
SparseVector *result;
|
||||
float *rx;
|
||||
int dim;
|
||||
int nnz = 0;
|
||||
int startIndex;
|
||||
|
||||
/* Indexing starts at 1, like substring */
|
||||
if (start < 1)
|
||||
start = 1;
|
||||
|
||||
if (end > a->dim)
|
||||
end = a->dim + 1;
|
||||
|
||||
dim = end - start;
|
||||
CheckDim(dim);
|
||||
|
||||
for (startIndex = 0; startIndex < a->nnz; startIndex++)
|
||||
{
|
||||
if (a->indices[startIndex] >= start - 1)
|
||||
break;
|
||||
}
|
||||
|
||||
for (int i = startIndex; i < a->nnz; i++)
|
||||
{
|
||||
if (a->indices[i] < end - 1)
|
||||
nnz++;
|
||||
}
|
||||
|
||||
result = InitSparseVector(dim, nnz);
|
||||
rx = SPARSEVEC_VALUES(result);
|
||||
|
||||
for (int i = 0; i < nnz; i++)
|
||||
{
|
||||
result->indices[i] = a->indices[startIndex + i];
|
||||
rx[i] = ax[startIndex + i];
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#define SPARSEVEC_MAX_DIM 100000
|
||||
|
||||
/* Ensure values are aligned */
|
||||
#define SPARSEVEC_SIZE(_nnz) (offsetof(SparseVector, indices) + MAXALIGN((_nnz) * sizeof(int32)) + (_nnz * sizeof(float)))
|
||||
#define SPARSEVEC_VALUES(x) ((float *) (((char *) (x)) + offsetof(SparseVector, indices) + MAXALIGN((x)->nnz * sizeof(int32))))
|
||||
#define DatumGetSparseVector(x) ((SparseVector *) PG_DETOAST_DATUM(x))
|
||||
|
||||
128
src/vector.c
128
src/vector.c
@@ -2,9 +2,11 @@
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include "bitvector.h"
|
||||
#include "catalog/pg_type.h"
|
||||
#include "common/shortest_dec.h"
|
||||
#include "fmgr.h"
|
||||
#include "halfvec.h"
|
||||
#include "hnsw.h"
|
||||
#include "ivfflat.h"
|
||||
#include "lib/stringinfo.h"
|
||||
@@ -198,6 +200,8 @@ vector_in(PG_FUNCTION_ARGS)
|
||||
|
||||
while (pt != NULL && *stringEnd != ']')
|
||||
{
|
||||
float val;
|
||||
|
||||
if (dim == VECTOR_MAX_DIM)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
||||
@@ -213,15 +217,23 @@ vector_in(PG_FUNCTION_ARGS)
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||
|
||||
/* Use strtof like float4in to avoid a double-rounding problem */
|
||||
x[dim] = strtof(pt, &stringEnd);
|
||||
CheckElement(x[dim]);
|
||||
dim++;
|
||||
errno = 0;
|
||||
val = strtof(pt, &stringEnd);
|
||||
|
||||
if (stringEnd == pt)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||
|
||||
/* Check for range error like float4in */
|
||||
if (errno == ERANGE && isinf(val))
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("\"%s\" is out of range for type vector", pt)));
|
||||
|
||||
CheckElement(val);
|
||||
x[dim++] = val;
|
||||
|
||||
while (vector_isspace(*stringEnd))
|
||||
stringEnd++;
|
||||
|
||||
@@ -276,6 +288,9 @@ vector_in(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
#define AppendChar(ptr, c) (*(ptr)++ = (c))
|
||||
#define AppendFloat(ptr, f) ((ptr) += float_to_shortest_decimal_bufn((f), (ptr)))
|
||||
|
||||
/*
|
||||
* Convert internal representation to textual representation
|
||||
*/
|
||||
@@ -287,7 +302,6 @@ vector_out(PG_FUNCTION_ARGS)
|
||||
int dim = vector->dim;
|
||||
char *buf;
|
||||
char *ptr;
|
||||
int n;
|
||||
|
||||
/*
|
||||
* Need:
|
||||
@@ -302,21 +316,17 @@ vector_out(PG_FUNCTION_ARGS)
|
||||
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
|
||||
ptr = buf;
|
||||
|
||||
*ptr = '[';
|
||||
ptr++;
|
||||
AppendChar(ptr, '[');
|
||||
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
if (i > 0)
|
||||
{
|
||||
*ptr = ',';
|
||||
ptr++;
|
||||
}
|
||||
AppendChar(ptr, ',');
|
||||
|
||||
n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
|
||||
ptr += n;
|
||||
AppendFloat(ptr, vector->x[i]);
|
||||
}
|
||||
*ptr = ']';
|
||||
ptr++;
|
||||
|
||||
AppendChar(ptr, ']');
|
||||
*ptr = '\0';
|
||||
|
||||
PG_FREE_IF_COPY(vector, 0);
|
||||
@@ -533,6 +543,28 @@ vector_to_float4(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert half vector to vector
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_vector);
|
||||
Datum
|
||||
halfvec_to_vector(PG_FUNCTION_ARGS)
|
||||
{
|
||||
HalfVector *vec = PG_GETARG_HALFVEC_P(0);
|
||||
int32 typmod = PG_GETARG_INT32(1);
|
||||
Vector *result;
|
||||
|
||||
CheckDim(vec->dim);
|
||||
CheckExpectedDim(typmod, vec->dim);
|
||||
|
||||
result = InitVector(vec->dim);
|
||||
|
||||
for (int i = 0; i < vec->dim; i++)
|
||||
result->x[i] = HalfToFloat4(vec->x[i]);
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the L2 distance between vectors
|
||||
*/
|
||||
@@ -861,6 +893,56 @@ vector_mul(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Quantize a vector
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(quantize_binary);
|
||||
Datum
|
||||
quantize_binary(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
float *ax = a->x;
|
||||
VarBit *result = InitBitVector(a->dim);
|
||||
unsigned char *rx = VARBITS(result);
|
||||
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
|
||||
|
||||
PG_RETURN_VARBIT_P(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a subvector
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
|
||||
Datum
|
||||
subvector(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
int32 start = PG_GETARG_INT32(1);
|
||||
int32 count = PG_GETARG_INT32(2);
|
||||
int32 end = start + count;
|
||||
float *ax = a->x;
|
||||
Vector *result;
|
||||
int dim;
|
||||
|
||||
/* Indexing starts at 1, like substring */
|
||||
if (start < 1)
|
||||
start = 1;
|
||||
|
||||
if (end > a->dim)
|
||||
end = a->dim + 1;
|
||||
|
||||
dim = end - start;
|
||||
CheckDim(dim);
|
||||
result = InitVector(dim);
|
||||
|
||||
for (int i = 0; i < dim; i++)
|
||||
result->x[i] = ax[start - 1 + i];
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Internal helper to compare vectors
|
||||
*/
|
||||
@@ -898,9 +980,6 @@ vector_lt(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
|
||||
}
|
||||
|
||||
@@ -914,9 +993,6 @@ vector_le(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
|
||||
}
|
||||
|
||||
@@ -930,9 +1006,6 @@ vector_eq(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
|
||||
}
|
||||
|
||||
@@ -946,9 +1019,6 @@ vector_ne(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
|
||||
}
|
||||
|
||||
@@ -962,9 +1032,6 @@ vector_ge(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
|
||||
}
|
||||
|
||||
@@ -978,9 +1045,6 @@ vector_gt(PG_FUNCTION_ARGS)
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
|
||||
/* TODO Remove in 0.7.0 */
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
|
||||
}
|
||||
|
||||
|
||||
64
test/expected/bit_functions.out
Normal file
64
test/expected/bit_functions.out
Normal file
@@ -0,0 +1,64 @@
|
||||
SELECT hamming_distance(B'111', B'111');
|
||||
hamming_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance(B'111', B'110');
|
||||
hamming_distance
|
||||
------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance(B'111', B'100');
|
||||
hamming_distance
|
||||
------------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance(B'111', B'000');
|
||||
hamming_distance
|
||||
------------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance(B'111', B'00');
|
||||
ERROR: different bit lengths 3 and 2
|
||||
SELECT jaccard_distance(B'1111', B'1111');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance(B'1111', B'1110');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0.25
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance(B'1111', B'1100');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0.5
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance(B'1111', B'1000');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0.75
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance(B'1111', B'0000');
|
||||
jaccard_distance
|
||||
------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance(B'1100', B'1000');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0.5
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance(B'1111', B'000');
|
||||
ERROR: different bit lengths 4 and 3
|
||||
@@ -46,6 +46,30 @@ SELECT '[1,2,3]'::vector::real[];
|
||||
{1,2,3}
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector::halfvec;
|
||||
halfvec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::halfvec::vector;
|
||||
vector
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector::halfvec(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '[1,2,3]'::halfvec::vector(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '[65520]'::vector::halfvec;
|
||||
ERROR: infinite value not allowed in halfvec
|
||||
SELECT '[1e-8]'::vector::halfvec;
|
||||
halfvec
|
||||
---------
|
||||
[0]
|
||||
(1 row)
|
||||
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
CREATE TABLE t (val vector(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE TABLE t2 (val vector(3));
|
||||
CREATE TABLE t (val vector(3), val2 halfvec(3));
|
||||
INSERT INTO t (val, val2) VALUES ('[0,0,0]', '[0,0,0]'), ('[1,2,3]', '[1,2,3]'), ('[1,1,1]', '[1,1,1]'), (NULL, NULL);
|
||||
CREATE TABLE t2 (val vector(3), val2 halfvec(3));
|
||||
\copy t TO 'results/data.bin' WITH (FORMAT binary)
|
||||
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
|
||||
SELECT * FROM t2 ORDER BY val;
|
||||
val
|
||||
---------
|
||||
[0,0,0]
|
||||
[1,1,1]
|
||||
[1,2,3]
|
||||
|
||||
val | val2
|
||||
---------+---------
|
||||
[0,0,0] | [0,0,0]
|
||||
[1,1,1] | [1,1,1]
|
||||
[1,2,3] | [1,2,3]
|
||||
|
|
||||
(4 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
146
test/expected/halfvec_functions.out
Normal file
146
test/expected/halfvec_functions.out
Normal file
@@ -0,0 +1,146 @@
|
||||
SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
|
||||
l2_distance
|
||||
-------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[0,0]'::halfvec, '[0,1]');
|
||||
l2_distance
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[1,2]'::halfvec, '[3]');
|
||||
ERROR: different halfvec dimensions 2 and 1
|
||||
SELECT '[0,0]'::halfvec <-> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]'::halfvec, '[3,4]');
|
||||
inner_product
|
||||
---------------
|
||||
11
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]'::halfvec, '[3]');
|
||||
ERROR: different halfvec dimensions 2 and 1
|
||||
SELECT inner_product('[65504]'::halfvec, '[65504]');
|
||||
inner_product
|
||||
---------------
|
||||
4290774016
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::halfvec <#> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
-11
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::halfvec, '[2,4]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::halfvec, '[0,0]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[1,1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,0]'::halfvec, '[0,2]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[-1,-1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::halfvec, '[3]');
|
||||
ERROR: different halfvec dimensions 2 and 1
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[1.1,1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[-1.1,-1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::halfvec <=> '[2,4]';
|
||||
?column?
|
||||
----------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
|
||||
l1_distance
|
||||
-------------
|
||||
7
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
|
||||
l1_distance
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[1,2]'::halfvec, '[3]');
|
||||
ERROR: different halfvec dimensions 2 and 1
|
||||
SELECT quantize_binary('[1,0,-1]'::halfvec);
|
||||
quantize_binary
|
||||
-----------------
|
||||
100
|
||||
(1 row)
|
||||
|
||||
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
|
||||
quantize_binary
|
||||
-----------------
|
||||
01001110101
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
|
||||
subvector
|
||||
-----------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
|
||||
subvector
|
||||
-----------
|
||||
[3,4]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 3);
|
||||
subvector
|
||||
-----------
|
||||
[1]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
|
||||
subvector
|
||||
-----------
|
||||
[3,4,5]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
|
||||
ERROR: halfvec must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
|
||||
ERROR: halfvec must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
|
||||
ERROR: halfvec must have at least 1 dimension
|
||||
147
test/expected/halfvec_input.out
Normal file
147
test/expected/halfvec_input.out
Normal file
@@ -0,0 +1,147 @@
|
||||
SELECT '[1,2,3]'::halfvec;
|
||||
halfvec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[-1,-2,-3]'::halfvec;
|
||||
halfvec
|
||||
------------
|
||||
[-1,-2,-3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.,2.,3.]'::halfvec;
|
||||
halfvec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT ' [ 1, 2 , 3 ] '::halfvec;
|
||||
halfvec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.23456]'::halfvec;
|
||||
halfvec
|
||||
------------
|
||||
[1.234375]
|
||||
(1 row)
|
||||
|
||||
SELECT '[hello,1]'::halfvec;
|
||||
ERROR: invalid input syntax for type halfvec: "[hello,1]"
|
||||
LINE 1: SELECT '[hello,1]'::halfvec;
|
||||
^
|
||||
SELECT '[NaN,1]'::halfvec;
|
||||
ERROR: NaN not allowed in halfvec
|
||||
LINE 1: SELECT '[NaN,1]'::halfvec;
|
||||
^
|
||||
SELECT '[Infinity,1]'::halfvec;
|
||||
ERROR: infinite value not allowed in halfvec
|
||||
LINE 1: SELECT '[Infinity,1]'::halfvec;
|
||||
^
|
||||
SELECT '[-Infinity,1]'::halfvec;
|
||||
ERROR: infinite value not allowed in halfvec
|
||||
LINE 1: SELECT '[-Infinity,1]'::halfvec;
|
||||
^
|
||||
SELECT '[65519,-65519]'::halfvec;
|
||||
halfvec
|
||||
----------------
|
||||
[65504,-65504]
|
||||
(1 row)
|
||||
|
||||
SELECT '[65520,-65520]'::halfvec;
|
||||
ERROR: value out of range: overflow
|
||||
LINE 1: SELECT '[65520,-65520]'::halfvec;
|
||||
^
|
||||
SELECT '[1e-8,-1e-8]'::halfvec;
|
||||
ERROR: value out of range: underflow
|
||||
LINE 1: SELECT '[1e-8,-1e-8]'::halfvec;
|
||||
^
|
||||
SELECT '[4e38,1]'::halfvec;
|
||||
ERROR: infinite value not allowed in halfvec
|
||||
LINE 1: SELECT '[4e38,1]'::halfvec;
|
||||
^
|
||||
SELECT '[1,2,3'::halfvec;
|
||||
ERROR: malformed halfvec literal: "[1,2,3"
|
||||
LINE 1: SELECT '[1,2,3'::halfvec;
|
||||
^
|
||||
DETAIL: Unexpected end of input.
|
||||
SELECT '[1,2,3]9'::halfvec;
|
||||
ERROR: malformed halfvec literal: "[1,2,3]9"
|
||||
LINE 1: SELECT '[1,2,3]9'::halfvec;
|
||||
^
|
||||
DETAIL: Junk after closing right brace.
|
||||
SELECT '1,2,3'::halfvec;
|
||||
ERROR: malformed halfvec literal: "1,2,3"
|
||||
LINE 1: SELECT '1,2,3'::halfvec;
|
||||
^
|
||||
DETAIL: Vector contents must start with "[".
|
||||
SELECT ''::halfvec;
|
||||
ERROR: malformed halfvec literal: ""
|
||||
LINE 1: SELECT ''::halfvec;
|
||||
^
|
||||
DETAIL: Vector contents must start with "[".
|
||||
SELECT '['::halfvec;
|
||||
ERROR: malformed halfvec literal: "["
|
||||
LINE 1: SELECT '['::halfvec;
|
||||
^
|
||||
DETAIL: Unexpected end of input.
|
||||
SELECT '[,'::halfvec;
|
||||
ERROR: malformed halfvec literal: "[,"
|
||||
LINE 1: SELECT '[,'::halfvec;
|
||||
^
|
||||
DETAIL: Unexpected end of input.
|
||||
SELECT '[]'::halfvec;
|
||||
ERROR: halfvec must have at least 1 dimension
|
||||
LINE 1: SELECT '[]'::halfvec;
|
||||
^
|
||||
SELECT '[1,]'::halfvec;
|
||||
ERROR: invalid input syntax for type halfvec: "[1,]"
|
||||
LINE 1: SELECT '[1,]'::halfvec;
|
||||
^
|
||||
SELECT '[1a]'::halfvec;
|
||||
ERROR: invalid input syntax for type halfvec: "[1a]"
|
||||
LINE 1: SELECT '[1a]'::halfvec;
|
||||
^
|
||||
SELECT '[1,,3]'::halfvec;
|
||||
ERROR: malformed halfvec literal: "[1,,3]"
|
||||
LINE 1: SELECT '[1,,3]'::halfvec;
|
||||
^
|
||||
SELECT '[1, ,3]'::halfvec;
|
||||
ERROR: invalid input syntax for type halfvec: "[1, ,3]"
|
||||
LINE 1: SELECT '[1, ,3]'::halfvec;
|
||||
^
|
||||
SELECT '[1,2,3]'::halfvec(3);
|
||||
halfvec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::halfvec(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '[1,2,3]'::halfvec(3, 2);
|
||||
ERROR: invalid type modifier
|
||||
LINE 1: SELECT '[1,2,3]'::halfvec(3, 2);
|
||||
^
|
||||
SELECT '[1,2,3]'::halfvec('a');
|
||||
ERROR: invalid input syntax for type integer: "a"
|
||||
LINE 1: SELECT '[1,2,3]'::halfvec('a');
|
||||
^
|
||||
SELECT '[1,2,3]'::halfvec(0);
|
||||
ERROR: dimensions for type halfvec must be at least 1
|
||||
LINE 1: SELECT '[1,2,3]'::halfvec(0);
|
||||
^
|
||||
SELECT '[1,2,3]'::halfvec(16001);
|
||||
ERROR: dimensions for type halfvec cannot exceed 16000
|
||||
LINE 1: SELECT '[1,2,3]'::halfvec(16001);
|
||||
^
|
||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
|
||||
unnest
|
||||
---------
|
||||
[1,2,3]
|
||||
[4,5,6]
|
||||
(2 rows)
|
||||
|
||||
SELECT '{"[1,2,3]"}'::halfvec(2)[];
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
21
test/expected/hnsw_bit_hamming.out
Normal file
21
test/expected/hnsw_bit_hamming.out
Normal file
@@ -0,0 +1,21 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val bit(3));
|
||||
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
|
||||
INSERT INTO t (val) VALUES (B'110');
|
||||
SELECT * FROM t ORDER BY val <~> B'111';
|
||||
val
|
||||
-----
|
||||
111
|
||||
110
|
||||
100
|
||||
000
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
21
test/expected/hnsw_bit_jaccard.out
Normal file
21
test/expected/hnsw_bit_jaccard.out
Normal file
@@ -0,0 +1,21 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val bit(4));
|
||||
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
|
||||
INSERT INTO t (val) VALUES (B'1110');
|
||||
SELECT * FROM t ORDER BY val <%> B'1111';
|
||||
val
|
||||
------
|
||||
1111
|
||||
1110
|
||||
1100
|
||||
0000
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
26
test/expected/hnsw_halfvec_cosine.out
Normal file
26
test/expected/hnsw_halfvec_cosine.out
Normal file
@@ -0,0 +1,26 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,1,1]
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
(3 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
|
||||
count
|
||||
-------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
|
||||
count
|
||||
-------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
21
test/expected/hnsw_halfvec_ip.out
Normal file
21
test/expected/hnsw_halfvec_ip.out
Normal file
@@ -0,0 +1,21 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,4]
|
||||
[1,2,3]
|
||||
[1,1,1]
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
33
test/expected/hnsw_halfvec_l2.out
Normal file
33
test/expected/hnsw_halfvec_l2.out
Normal file
@@ -0,0 +1,33 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
[1,1,1]
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
SELECT COUNT(*) FROM t;
|
||||
count
|
||||
-------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
TRUNCATE t;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -60,3 +60,33 @@ SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
|
||||
|
||||
SELECT cosine_distance('{0:1}/2'::sparsevec, '{0:1}/3');
|
||||
ERROR: different sparsevec dimensions 2 and 3
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 1, 3);
|
||||
subvector
|
||||
-----------------
|
||||
{0:1,1:2,2:3}/3
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 3, 2);
|
||||
subvector
|
||||
-------------
|
||||
{2:3,3:4}/2
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, -1, 3);
|
||||
subvector
|
||||
-----------
|
||||
{0:1}/1
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 3, 9);
|
||||
subvector
|
||||
-----------------
|
||||
{2:3,3:4,4:5}/3
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 1, 0);
|
||||
ERROR: sparsevec must have at least 1 dimension
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 3, -1);
|
||||
ERROR: sparsevec must have at least 1 dimension
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, -1, 2);
|
||||
ERROR: sparsevec must have at least 1 dimension
|
||||
|
||||
@@ -24,6 +24,30 @@ SELECT '[1e37]'::vector * '[1e37]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[1e-37]'::vector * '[1e-37]';
|
||||
ERROR: value out of range: underflow
|
||||
SELECT '[1,2,3]'::vector < '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector < '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector <= '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector <= '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
@@ -31,7 +55,47 @@ SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector = '[1,2]';
|
||||
ERROR: different vector dimensions 3 and 2
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector != '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector != '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector >= '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector >= '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector > '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector > '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
|
||||
vector_cmp
|
||||
------------
|
||||
@@ -208,6 +272,48 @@ SELECT l1_distance('[3e38]'::vector, '[-3e38]');
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT quantize_binary('[1,0,-1]'::vector);
|
||||
quantize_binary
|
||||
-----------------
|
||||
100
|
||||
(1 row)
|
||||
|
||||
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
|
||||
quantize_binary
|
||||
-----------------
|
||||
01001110101
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
|
||||
subvector
|
||||
-----------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
|
||||
subvector
|
||||
-----------
|
||||
[3,4]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, -1, 3);
|
||||
subvector
|
||||
-----------
|
||||
[1]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, 9);
|
||||
subvector
|
||||
-----------
|
||||
[3,4,5]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 1, 0);
|
||||
ERROR: vector must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
|
||||
ERROR: vector must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
|
||||
ERROR: vector must have at least 1 dimension
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
avg
|
||||
-----------
|
||||
@@ -63,11 +63,11 @@ SELECT '[1.5e-38,-1.5e-38]'::vector;
|
||||
(1 row)
|
||||
|
||||
SELECT '[4e38,1]'::vector;
|
||||
ERROR: infinite value not allowed in vector
|
||||
ERROR: "4e38" is out of range for type vector
|
||||
LINE 1: SELECT '[4e38,1]'::vector;
|
||||
^
|
||||
SELECT '[-4e38,1]'::vector;
|
||||
ERROR: infinite value not allowed in vector
|
||||
ERROR: "-4e38" is out of range for type vector
|
||||
LINE 1: SELECT '[-4e38,1]'::vector;
|
||||
^
|
||||
SELECT '[1e-46,1]'::vector;
|
||||
13
test/sql/bit_functions.sql
Normal file
13
test/sql/bit_functions.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
SELECT hamming_distance(B'111', B'111');
|
||||
SELECT hamming_distance(B'111', B'110');
|
||||
SELECT hamming_distance(B'111', B'100');
|
||||
SELECT hamming_distance(B'111', B'000');
|
||||
SELECT hamming_distance(B'111', B'00');
|
||||
|
||||
SELECT jaccard_distance(B'1111', B'1111');
|
||||
SELECT jaccard_distance(B'1111', B'1110');
|
||||
SELECT jaccard_distance(B'1111', B'1100');
|
||||
SELECT jaccard_distance(B'1111', B'1000');
|
||||
SELECT jaccard_distance(B'1111', B'0000');
|
||||
SELECT jaccard_distance(B'1100', B'1000');
|
||||
SELECT jaccard_distance(B'1111', B'000');
|
||||
@@ -10,6 +10,12 @@ SELECT '{-Infinity}'::real[]::vector;
|
||||
SELECT '{}'::real[]::vector;
|
||||
SELECT '{{1}}'::real[]::vector;
|
||||
SELECT '[1,2,3]'::vector::real[];
|
||||
SELECT '[1,2,3]'::vector::halfvec;
|
||||
SELECT '[1,2,3]'::halfvec::vector;
|
||||
SELECT '[1,2,3]'::vector::halfvec(2);
|
||||
SELECT '[1,2,3]'::halfvec::vector(2);
|
||||
SELECT '[65520]'::vector::halfvec;
|
||||
SELECT '[1e-8]'::vector::halfvec;
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
CREATE TABLE t (val vector(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE TABLE t (val vector(3), val2 halfvec(3));
|
||||
INSERT INTO t (val, val2) VALUES ('[0,0,0]', '[0,0,0]'), ('[1,2,3]', '[1,2,3]'), ('[1,1,1]', '[1,1,1]'), (NULL, NULL);
|
||||
|
||||
CREATE TABLE t2 (val vector(3));
|
||||
CREATE TABLE t2 (val vector(3), val2 halfvec(3));
|
||||
|
||||
\copy t TO 'results/data.bin' WITH (FORMAT binary)
|
||||
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
|
||||
|
||||
34
test/sql/halfvec_functions.sql
Normal file
34
test/sql/halfvec_functions.sql
Normal file
@@ -0,0 +1,34 @@
|
||||
SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
|
||||
SELECT l2_distance('[0,0]'::halfvec, '[0,1]');
|
||||
SELECT l2_distance('[1,2]'::halfvec, '[3]');
|
||||
SELECT '[0,0]'::halfvec <-> '[3,4]';
|
||||
|
||||
SELECT inner_product('[1,2]'::halfvec, '[3,4]');
|
||||
SELECT inner_product('[1,2]'::halfvec, '[3]');
|
||||
SELECT inner_product('[65504]'::halfvec, '[65504]');
|
||||
SELECT '[1,2]'::halfvec <#> '[3,4]';
|
||||
|
||||
SELECT cosine_distance('[1,2]'::halfvec, '[2,4]');
|
||||
SELECT cosine_distance('[1,2]'::halfvec, '[0,0]');
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[1,1]');
|
||||
SELECT cosine_distance('[1,0]'::halfvec, '[0,2]');
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[-1,-1]');
|
||||
SELECT cosine_distance('[1,2]'::halfvec, '[3]');
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]'::halfvec, '[-1.1,-1.1]');
|
||||
SELECT '[1,2]'::halfvec <=> '[2,4]';
|
||||
|
||||
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
|
||||
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
|
||||
SELECT l1_distance('[1,2]'::halfvec, '[3]');
|
||||
|
||||
SELECT quantize_binary('[1,0,-1]'::halfvec);
|
||||
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
|
||||
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
|
||||
34
test/sql/halfvec_input.sql
Normal file
34
test/sql/halfvec_input.sql
Normal file
@@ -0,0 +1,34 @@
|
||||
SELECT '[1,2,3]'::halfvec;
|
||||
SELECT '[-1,-2,-3]'::halfvec;
|
||||
SELECT '[1.,2.,3.]'::halfvec;
|
||||
SELECT ' [ 1, 2 , 3 ] '::halfvec;
|
||||
SELECT '[1.23456]'::halfvec;
|
||||
SELECT '[hello,1]'::halfvec;
|
||||
SELECT '[NaN,1]'::halfvec;
|
||||
SELECT '[Infinity,1]'::halfvec;
|
||||
SELECT '[-Infinity,1]'::halfvec;
|
||||
SELECT '[65519,-65519]'::halfvec;
|
||||
SELECT '[65520,-65520]'::halfvec;
|
||||
SELECT '[1e-8,-1e-8]'::halfvec;
|
||||
SELECT '[4e38,1]'::halfvec;
|
||||
SELECT '[1,2,3'::halfvec;
|
||||
SELECT '[1,2,3]9'::halfvec;
|
||||
SELECT '1,2,3'::halfvec;
|
||||
SELECT ''::halfvec;
|
||||
SELECT '['::halfvec;
|
||||
SELECT '[,'::halfvec;
|
||||
SELECT '[]'::halfvec;
|
||||
SELECT '[1,]'::halfvec;
|
||||
SELECT '[1a]'::halfvec;
|
||||
SELECT '[1,,3]'::halfvec;
|
||||
SELECT '[1, ,3]'::halfvec;
|
||||
|
||||
SELECT '[1,2,3]'::halfvec(3);
|
||||
SELECT '[1,2,3]'::halfvec(2);
|
||||
SELECT '[1,2,3]'::halfvec(3, 2);
|
||||
SELECT '[1,2,3]'::halfvec('a');
|
||||
SELECT '[1,2,3]'::halfvec(0);
|
||||
SELECT '[1,2,3]'::halfvec(16001);
|
||||
|
||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
|
||||
SELECT '{"[1,2,3]"}'::halfvec(2)[];
|
||||
12
test/sql/hnsw_bit_hamming.sql
Normal file
12
test/sql/hnsw_bit_hamming.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val bit(3));
|
||||
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES (B'110');
|
||||
|
||||
SELECT * FROM t ORDER BY val <~> B'111';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
12
test/sql/hnsw_bit_jaccard.sql
Normal file
12
test/sql/hnsw_bit_jaccard.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val bit(4));
|
||||
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES (B'1110');
|
||||
|
||||
SELECT * FROM t ORDER BY val <%> B'1111';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
13
test/sql/hnsw_halfvec_cosine.sql
Normal file
13
test/sql/hnsw_halfvec_cosine.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
12
test/sql/hnsw_halfvec_ip.sql
Normal file
12
test/sql/hnsw_halfvec_ip.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
16
test/sql/hnsw_halfvec_l2.sql
Normal file
16
test/sql/hnsw_halfvec_l2.sql
Normal file
@@ -0,0 +1,16 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
|
||||
SELECT COUNT(*) FROM t;
|
||||
|
||||
TRUNCATE t;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -11,3 +11,11 @@ SELECT cosine_distance('{0:1,1:1}/2'::sparsevec, '{0:-1,1:-1}/2');
|
||||
SELECT cosine_distance('{0:1}/2'::sparsevec, '{1:2}/2');
|
||||
SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
|
||||
SELECT cosine_distance('{0:1}/2'::sparsevec, '{0:1}/3');
|
||||
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 1, 3);
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 3, 2);
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, -1, 3);
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 3, 9);
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 1, 0);
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, 3, -1);
|
||||
SELECT subvector('{0:1,1:2,2:3,3:4,4:5}/5'::sparsevec, -1, 2);
|
||||
|
||||
@@ -6,8 +6,18 @@ SELECT '[1,2,3]'::vector * '[4,5,6]';
|
||||
SELECT '[1e37]'::vector * '[1e37]';
|
||||
SELECT '[1e-37]'::vector * '[1e-37]';
|
||||
|
||||
SELECT '[1,2,3]'::vector < '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector < '[1,2]';
|
||||
SELECT '[1,2,3]'::vector <= '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector <= '[1,2]';
|
||||
SELECT '[1,2,3]'::vector = '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector = '[1,2]';
|
||||
SELECT '[1,2,3]'::vector != '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector != '[1,2]';
|
||||
SELECT '[1,2,3]'::vector >= '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector >= '[1,2]';
|
||||
SELECT '[1,2,3]'::vector > '[1,2,3]';
|
||||
SELECT '[1,2,3]'::vector > '[1,2]';
|
||||
|
||||
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
|
||||
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
|
||||
@@ -48,6 +58,17 @@ SELECT l1_distance('[0,0]'::vector, '[0,1]');
|
||||
SELECT l1_distance('[1,2]'::vector, '[3]');
|
||||
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
|
||||
|
||||
SELECT quantize_binary('[1,0,-1]'::vector);
|
||||
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, -1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, 9);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 1, 0);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
|
||||
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||
137
test/t/020_hnsw_bit_build_recall.pl
Normal file
137
test/t/020_hnsw_bit_build_recall.pl
Normal file
@@ -0,0 +1,137 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 52;
|
||||
my $max = 2**$dim;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = 100;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator $queries[0] LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = 100;
|
||||
SELECT i FROM tst ORDER BY v $operator $queries[$i] LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
my %expected_set = map { $_ => 1 } @expected_ids;
|
||||
|
||||
foreach (@actual_ids)
|
||||
{
|
||||
if (exists($expected_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
}
|
||||
|
||||
$total += $limit;
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v bit($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, (random() * $max)::bigint::bit($dim) FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r = int(rand() * $max);
|
||||
push(@queries, "${r}::bigint::bit($dim)");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<~>", "<\%>");
|
||||
my @opclasses = ("bit_hamming_ops", "bit_jaccard_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
# Handle ties
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
WITH top AS (
|
||||
SELECT v $operator $_ AS distance FROM tst ORDER BY v $operator $_ LIMIT $limit
|
||||
)
|
||||
SELECT i FROM tst WHERE (v $operator $_) <= (SELECT MAX(distance) FROM top)
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Build index serially
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET max_parallel_maintenance_workers = 0;
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
|
||||
# Test approximate results
|
||||
my $min = $operator eq "<\%>" ? 0.96 : 0.98;
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel in memory
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET client_min_messages = DEBUG;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
|
||||
# Test approximate results
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel on disk
|
||||
# Set parallel_workers on table to use workers with low maintenance_work_mem
|
||||
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
ALTER TABLE tst SET (parallel_workers = 2);
|
||||
SET client_min_messages = DEBUG;
|
||||
SET maintenance_work_mem = '4MB';
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
ALTER TABLE tst RESET (parallel_workers);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
132
test/t/021_hnsw_halfvec_build_recall.pl
Normal file
132
test/t/021_hnsw_halfvec_build_recall.pl
Normal file
@@ -0,0 +1,132 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 10;
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v halfvec($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
push(@queries, "[" . join(",", @r) . "]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("halfvec_l2_ops", "halfvec_ip_ops", "halfvec_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Build index serially
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET max_parallel_maintenance_workers = 0;
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
|
||||
# Test approximate results
|
||||
my $min = $operator eq "<#>" ? 0.95 : 0.98;
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel in memory
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET client_min_messages = DEBUG;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
|
||||
# Test approximate results
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel on disk
|
||||
# Set parallel_workers on table to use workers with low maintenance_work_mem
|
||||
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
ALTER TABLE tst SET (parallel_workers = 2);
|
||||
SET client_min_messages = DEBUG;
|
||||
SET maintenance_work_mem = '4MB';
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
ALTER TABLE tst RESET (parallel_workers);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
Reference in New Issue
Block a user