mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 20:15:46 +08:00
Compare commits
3 Commits
index-type
...
half-dispa
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9d5a0b1aa | ||
|
|
52a81b3b9d | ||
|
|
c5372e493b |
@@ -3,7 +3,6 @@
|
||||
- Added `halfvec` type
|
||||
- Added `sparsevec` type
|
||||
- Added support for indexing `bit` type
|
||||
- Added support for indexing L1 distance with HNSW
|
||||
- Added `binary_quantize` function
|
||||
- Added `hamming_distance` function
|
||||
- Added `jaccard_distance` function
|
||||
|
||||
2
Makefile
2
Makefile
@@ -3,7 +3,7 @@ EXTVERSION = 0.6.2
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
|
||||
OBJS = src/bitvector.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
|
||||
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
|
||||
|
||||
TESTS = $(wildcard test/sql/*.sql)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.6.2
|
||||
|
||||
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
OBJS = src\bitvector.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
|
||||
|
||||
REGRESS = bit_functions btree_halfvec btree_sparsevec btree_vector cast copy halfvec_functions halfvec_input hnsw_bit_hamming hnsw_bit_jaccard hnsw_halfvec_cosine hnsw_halfvec_ip hnsw_halfvec_l2 hnsw_options hnsw_sparsevec_cosine hnsw_sparsevec_ip hnsw_sparsevec_l2 hnsw_unlogged hnsw_vector_cosine hnsw_vector_ip hnsw_vector_l2 ivfflat_halfvec_cosine ivfflat_halfvec_ip ivfflat_halfvec_l2 ivfflat_options ivfflat_unlogged ivfflat_vector_cosine ivfflat_vector_ip ivfflat_vector_l2 sparsevec_functions sparsevec_input vector_functions vector_input
|
||||
|
||||
20
README.md
20
README.md
@@ -81,7 +81,7 @@ Get the nearest neighbors by L2 distance
|
||||
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, unreleased)
|
||||
Also supports inner product (`<#>`) and cosine distance (`<=>`)
|
||||
|
||||
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
|
||||
|
||||
@@ -143,7 +143,6 @@ Supported distance functions are:
|
||||
- `<->` - L2 distance
|
||||
- `<#>` - (negative) inner product
|
||||
- `<=>` - cosine distance
|
||||
- `<+>` - L1 distance (unreleased)
|
||||
|
||||
Get the nearest neighbors to a row
|
||||
|
||||
@@ -228,12 +227,6 @@ Cosine distance
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
L1 distance - unreleased
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
|
||||
```
|
||||
|
||||
Hamming distance - unreleased
|
||||
|
||||
```sql
|
||||
@@ -356,12 +349,6 @@ Cosine distance
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
Hamming distance - unreleased
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
|
||||
```
|
||||
|
||||
Supported types are:
|
||||
|
||||
- `vector` - up to 2,000 dimensions
|
||||
@@ -548,7 +535,7 @@ SELECT id, content FROM items, plainto_tsquery('hello search') query
|
||||
|
||||
You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search_rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py) to combine results.
|
||||
|
||||
## Indexing Subvectors
|
||||
## Subvector Indexing
|
||||
|
||||
*Unreleased*
|
||||
|
||||
@@ -868,7 +855,6 @@ Operator | Description | Added
|
||||
<-> | Euclidean distance |
|
||||
<#> | negative inner product |
|
||||
<=> | cosine distance |
|
||||
<+> | taxicab distance | unreleased
|
||||
|
||||
### Vector Functions
|
||||
|
||||
@@ -906,7 +892,6 @@ Operator | Description | Added
|
||||
<-> | Euclidean distance | unreleased
|
||||
<#> | negative inner product | unreleased
|
||||
<=> | cosine distance | unreleased
|
||||
<+> | taxicab distance | unreleased
|
||||
|
||||
### Halfvec Functions
|
||||
|
||||
@@ -958,7 +943,6 @@ Operator | Description | Added
|
||||
<-> | Euclidean distance | unreleased
|
||||
<#> | negative inner product | unreleased
|
||||
<=> | cosine distance | unreleased
|
||||
<+> | taxicab distance | unreleased
|
||||
|
||||
### Sparsevec Functions
|
||||
|
||||
|
||||
@@ -13,32 +13,16 @@ CREATE FUNCTION subvector(vector, int, int) RETURNS vector
|
||||
CREATE FUNCTION vector_concat(vector, vector) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR || (
|
||||
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_concat
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS vector_l1_ops
|
||||
FOR TYPE vector USING hnsw AS
|
||||
OPERATOR 1 <+> (vector, vector) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(vector, vector);
|
||||
|
||||
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION bit_ivfflat_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION bit_hnsw_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <~> (
|
||||
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
|
||||
COMMUTATOR = '<~>'
|
||||
@@ -53,20 +37,23 @@ CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit),
|
||||
FUNCTION 3 hamming_distance(bit, bit),
|
||||
FUNCTION 6 bit_ivfflat_support(internal);
|
||||
FUNCTION 3 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit),
|
||||
FUNCTION 3 jaccard_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit),
|
||||
FUNCTION 4 bit_hnsw_support(internal);
|
||||
FUNCTION 1 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit),
|
||||
FUNCTION 4 bit_hnsw_support(internal);
|
||||
FUNCTION 1 jaccard_distance(bit, bit);
|
||||
|
||||
CREATE TYPE halfvec;
|
||||
|
||||
@@ -169,12 +156,6 @@ CREATE FUNCTION halfvec_accum(double precision[], halfvec) RETURNS double precis
|
||||
CREATE FUNCTION halfvec_avg(double precision[]) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_ivfflat_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_hnsw_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE AGGREGATE avg(halfvec) (
|
||||
SFUNC = halfvec_accum,
|
||||
STYPE = double precision[],
|
||||
@@ -254,11 +235,6 @@ CREATE OPERATOR <=> (
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR + (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_add,
|
||||
COMMUTATOR = +
|
||||
@@ -326,17 +302,14 @@ CREATE OPERATOR CLASS halfvec_l2_ops
|
||||
FOR TYPE halfvec USING ivfflat AS
|
||||
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
|
||||
FUNCTION 3 l2_distance(halfvec, halfvec),
|
||||
FUNCTION 6 halfvec_ivfflat_support(internal);
|
||||
FUNCTION 3 l2_distance(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_ip_ops
|
||||
FOR TYPE halfvec USING ivfflat AS
|
||||
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
|
||||
FUNCTION 4 l2_norm(halfvec),
|
||||
FUNCTION 5 l2_normalize(halfvec),
|
||||
FUNCTION 6 halfvec_ivfflat_support(internal);
|
||||
FUNCTION 4 l2_norm(halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FOR TYPE halfvec USING ivfflat AS
|
||||
@@ -344,35 +317,23 @@ CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 2 l2_norm(halfvec),
|
||||
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
|
||||
FUNCTION 4 l2_norm(halfvec),
|
||||
FUNCTION 5 l2_normalize(halfvec),
|
||||
FUNCTION 6 halfvec_ivfflat_support(internal);
|
||||
FUNCTION 4 l2_norm(halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_l2_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_support(internal);
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_ip_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_support(internal);
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 2 l2_norm(halfvec),
|
||||
FUNCTION 3 l2_normalize(halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_l1_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <+> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(halfvec, halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_support(internal);
|
||||
FUNCTION 2 l2_norm(halfvec);
|
||||
|
||||
CREATE TYPE sparsevec;
|
||||
|
||||
@@ -454,15 +415,6 @@ CREATE FUNCTION vector_to_sparsevec(vector, integer, boolean) RETURNS sparsevec
|
||||
CREATE FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparsevec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION sparsevec_hnsw_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE CAST (sparsevec AS sparsevec)
|
||||
WITH FUNCTION sparsevec(sparsevec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
@@ -472,12 +424,6 @@ CREATE CAST (sparsevec AS vector)
|
||||
CREATE CAST (vector AS sparsevec)
|
||||
WITH FUNCTION vector_to_sparsevec(vector, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (sparsevec AS halfvec)
|
||||
WITH FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (halfvec AS sparsevec)
|
||||
WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
@@ -493,11 +439,6 @@ CREATE OPERATOR <=> (
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR < (
|
||||
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_lt,
|
||||
COMMUTATOR = > , NEGATOR = >= ,
|
||||
@@ -546,25 +487,15 @@ CREATE OPERATOR CLASS sparsevec_ops
|
||||
CREATE OPERATOR CLASS sparsevec_l2_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_support(internal);
|
||||
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec);
|
||||
|
||||
CREATE OPERATOR CLASS sparsevec_ip_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_support(internal);
|
||||
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec);
|
||||
|
||||
CREATE OPERATOR CLASS sparsevec_cosine_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
|
||||
FUNCTION 2 l2_norm(sparsevec),
|
||||
FUNCTION 3 l2_normalize(sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS sparsevec_l1_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(sparsevec, sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_support(internal);
|
||||
FUNCTION 2 l2_norm(sparsevec);
|
||||
|
||||
114
sql/vector.sql
114
sql/vector.sql
@@ -186,11 +186,6 @@ CREATE OPERATOR <=> (
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR + (
|
||||
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_add,
|
||||
COMMUTATOR = +
|
||||
@@ -309,11 +304,6 @@ CREATE OPERATOR CLASS vector_cosine_ops
|
||||
FUNCTION 1 vector_negative_inner_product(vector, vector),
|
||||
FUNCTION 2 vector_norm(vector);
|
||||
|
||||
CREATE OPERATOR CLASS vector_l1_ops
|
||||
FOR TYPE vector USING hnsw AS
|
||||
OPERATOR 1 <+> (vector, vector) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(vector, vector);
|
||||
|
||||
-- bit functions
|
||||
|
||||
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
@@ -322,14 +312,6 @@ CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- bit private functions
|
||||
|
||||
CREATE FUNCTION bit_ivfflat_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION bit_hnsw_max_dims(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- bit operators
|
||||
|
||||
CREATE OPERATOR <~> (
|
||||
@@ -348,20 +330,23 @@ CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit),
|
||||
FUNCTION 3 hamming_distance(bit, bit),
|
||||
FUNCTION 6 bit_ivfflat_support(internal);
|
||||
FUNCTION 3 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit),
|
||||
FUNCTION 3 jaccard_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit),
|
||||
FUNCTION 4 bit_hnsw_max_dims(internal);
|
||||
FUNCTION 1 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit),
|
||||
FUNCTION 4 bit_hnsw_max_dims(internal);
|
||||
FUNCTION 1 jaccard_distance(bit, bit);
|
||||
|
||||
-- halfvec type
|
||||
|
||||
@@ -470,12 +455,6 @@ CREATE FUNCTION halfvec_accum(double precision[], halfvec) RETURNS double precis
|
||||
CREATE FUNCTION halfvec_avg(double precision[]) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_ivfflat_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_hnsw_max_dims(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- halfvec aggregates
|
||||
|
||||
CREATE AGGREGATE avg(halfvec) (
|
||||
@@ -563,11 +542,6 @@ CREATE OPERATOR <=> (
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR + (
|
||||
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_add,
|
||||
COMMUTATOR = +
|
||||
@@ -637,17 +611,14 @@ CREATE OPERATOR CLASS halfvec_l2_ops
|
||||
FOR TYPE halfvec USING ivfflat AS
|
||||
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
|
||||
FUNCTION 3 l2_distance(halfvec, halfvec),
|
||||
FUNCTION 6 halfvec_ivfflat_support(internal);
|
||||
FUNCTION 3 l2_distance(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_ip_ops
|
||||
FOR TYPE halfvec USING ivfflat AS
|
||||
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
|
||||
FUNCTION 4 l2_norm(halfvec),
|
||||
FUNCTION 5 l2_normalize(halfvec),
|
||||
FUNCTION 6 halfvec_ivfflat_support(internal);
|
||||
FUNCTION 4 l2_norm(halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FOR TYPE halfvec USING ivfflat AS
|
||||
@@ -655,35 +626,23 @@ CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 2 l2_norm(halfvec),
|
||||
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
|
||||
FUNCTION 4 l2_norm(halfvec),
|
||||
FUNCTION 5 l2_normalize(halfvec),
|
||||
FUNCTION 6 halfvec_ivfflat_support(internal);
|
||||
FUNCTION 4 l2_norm(halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_l2_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_max_dims(internal);
|
||||
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_ip_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_max_dims(internal);
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_cosine_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
|
||||
FUNCTION 2 l2_norm(halfvec),
|
||||
FUNCTION 3 l2_normalize(halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_max_dims(internal);
|
||||
|
||||
CREATE OPERATOR CLASS halfvec_l1_ops
|
||||
FOR TYPE halfvec USING hnsw AS
|
||||
OPERATOR 1 <+> (halfvec, halfvec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(halfvec, halfvec),
|
||||
FUNCTION 4 halfvec_hnsw_max_dims(internal);
|
||||
FUNCTION 2 l2_norm(halfvec);
|
||||
|
||||
--- sparsevec type
|
||||
|
||||
@@ -773,18 +732,6 @@ CREATE FUNCTION vector_to_sparsevec(vector, integer, boolean) RETURNS sparsevec
|
||||
CREATE FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparsevec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION sparsevec_hnsw_max_dims(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION sparsevec_hnsw_check_value(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- sparsevec casts
|
||||
|
||||
CREATE CAST (sparsevec AS sparsevec)
|
||||
@@ -796,12 +743,6 @@ CREATE CAST (sparsevec AS vector)
|
||||
CREATE CAST (vector AS sparsevec)
|
||||
WITH FUNCTION vector_to_sparsevec(vector, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (sparsevec AS halfvec)
|
||||
WITH FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (halfvec AS sparsevec)
|
||||
WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
-- sparsevec operators
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
@@ -819,11 +760,6 @@ CREATE OPERATOR <=> (
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR < (
|
||||
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_lt,
|
||||
COMMUTATOR = > , NEGATOR = >= ,
|
||||
@@ -874,29 +810,15 @@ CREATE OPERATOR CLASS sparsevec_ops
|
||||
CREATE OPERATOR CLASS sparsevec_l2_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_max_dims(internal),
|
||||
FUNCTION 5 sparsevec_hnsw_check_value(internal);
|
||||
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec);
|
||||
|
||||
CREATE OPERATOR CLASS sparsevec_ip_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_max_dims(internal),
|
||||
FUNCTION 5 sparsevec_hnsw_check_value(internal);
|
||||
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec);
|
||||
|
||||
CREATE OPERATOR CLASS sparsevec_cosine_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
|
||||
FUNCTION 2 l2_norm(sparsevec),
|
||||
FUNCTION 3 l2_normalize(sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_max_dims(internal),
|
||||
FUNCTION 5 sparsevec_hnsw_check_value(internal);
|
||||
|
||||
CREATE OPERATOR CLASS sparsevec_l1_ops
|
||||
FOR TYPE sparsevec USING hnsw AS
|
||||
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(sparsevec, sparsevec),
|
||||
FUNCTION 4 sparsevec_hnsw_max_dims(internal),
|
||||
FUNCTION 5 sparsevec_hnsw_check_value(internal);
|
||||
FUNCTION 2 l2_norm(sparsevec);
|
||||
|
||||
217
src/bitutils.c
217
src/bitutils.c
@@ -1,217 +0,0 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include "bitutils.h"
|
||||
#include "port/pg_bitutils.h"
|
||||
|
||||
#ifdef BIT_DISPATCH
|
||||
#include <immintrin.h>
|
||||
|
||||
#if defined(HAVE__GET_CPUID)
|
||||
#include <cpuid.h>
|
||||
#else
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define TARGET_AVX512_POPCOUNT
|
||||
#else
|
||||
#define TARGET_AVX512_POPCOUNT __attribute__((target("avx512f,avx512vpopcntdq")))
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Disable for LLVM due to crash with bitcode generation */
|
||||
#if defined(USE_TARGET_CLONES) && !defined(__POPCNT__) && !defined(__llvm__)
|
||||
#define BIT_TARGET_CLONES __attribute__((target_clones("default", "popcnt")))
|
||||
#else
|
||||
#define BIT_TARGET_CLONES
|
||||
#endif
|
||||
|
||||
/* Use built-ins when possible for inlining */
|
||||
#if defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_INT_64)
|
||||
#define popcount64(x) __builtin_popcountl(x)
|
||||
#elif defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_LONG_INT_64)
|
||||
#define popcount64(x) __builtin_popcountll(x)
|
||||
#elif !defined(_MSC_VER)
|
||||
/* Fails to resolve with MSVC */
|
||||
#define popcount64(x) pg_popcount64(x)
|
||||
#endif
|
||||
|
||||
uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);
|
||||
double (*BitJaccardDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb);
|
||||
|
||||
BIT_TARGET_CLONES static uint64
|
||||
BitHammingDistanceDefault(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance)
|
||||
{
|
||||
#ifdef popcount64
|
||||
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
|
||||
{
|
||||
uint64 axs;
|
||||
uint64 bxs;
|
||||
|
||||
/* Ensure aligned */
|
||||
memcpy(&axs, ax, sizeof(uint64));
|
||||
memcpy(&bxs, bx, sizeof(uint64));
|
||||
|
||||
distance += popcount64(axs ^ bxs);
|
||||
|
||||
ax += sizeof(uint64);
|
||||
bx += sizeof(uint64);
|
||||
}
|
||||
#endif
|
||||
|
||||
for (uint32 i = 0; i < bytes; i++)
|
||||
distance += pg_number_of_ones[ax[i] ^ bx[i]];
|
||||
|
||||
return distance;
|
||||
}
|
||||
|
||||
#ifdef BIT_DISPATCH
|
||||
TARGET_AVX512_POPCOUNT static uint64
|
||||
BitHammingDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance)
|
||||
{
|
||||
__m512i dist = _mm512_setzero_si512();
|
||||
|
||||
for (; bytes >= sizeof(__m512i); bytes -= sizeof(__m512i))
|
||||
{
|
||||
__m512i axs = _mm512_loadu_si512((const __m512i *) ax);
|
||||
__m512i bxs = _mm512_loadu_si512((const __m512i *) bx);
|
||||
|
||||
dist = _mm512_add_epi64(dist, _mm512_popcnt_epi64(_mm512_xor_si512(axs, bxs)));
|
||||
|
||||
ax += sizeof(__m512i);
|
||||
bx += sizeof(__m512i);
|
||||
}
|
||||
|
||||
distance += _mm512_reduce_add_epi64(dist);
|
||||
|
||||
return BitHammingDistanceDefault(bytes, ax, bx, distance);
|
||||
}
|
||||
#endif
|
||||
|
||||
BIT_TARGET_CLONES static double
|
||||
BitJaccardDistanceDefault(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb)
|
||||
{
|
||||
#ifdef popcount64
|
||||
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
|
||||
{
|
||||
uint64 axs;
|
||||
uint64 bxs;
|
||||
|
||||
/* Ensure aligned */
|
||||
memcpy(&axs, ax, sizeof(uint64));
|
||||
memcpy(&bxs, bx, sizeof(uint64));
|
||||
|
||||
ab += popcount64(axs & bxs);
|
||||
aa += popcount64(axs);
|
||||
bb += popcount64(bxs);
|
||||
|
||||
ax += sizeof(uint64);
|
||||
bx += sizeof(uint64);
|
||||
}
|
||||
#endif
|
||||
|
||||
for (uint32 i = 0; i < bytes; i++)
|
||||
{
|
||||
ab += pg_number_of_ones[ax[i] & bx[i]];
|
||||
aa += pg_number_of_ones[ax[i]];
|
||||
bb += pg_number_of_ones[bx[i]];
|
||||
}
|
||||
|
||||
if (ab == 0)
|
||||
return 1;
|
||||
else
|
||||
return 1 - (ab / ((double) (aa + bb - ab)));
|
||||
}
|
||||
|
||||
#ifdef BIT_DISPATCH
|
||||
TARGET_AVX512_POPCOUNT static double
|
||||
BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb)
|
||||
{
|
||||
__m512i abx = _mm512_setzero_si512();
|
||||
__m512i aax = _mm512_setzero_si512();
|
||||
__m512i bbx = _mm512_setzero_si512();
|
||||
|
||||
for (; bytes >= sizeof(__m512i); bytes -= sizeof(__m512i))
|
||||
{
|
||||
__m512i axs = _mm512_loadu_si512((const __m512i *) ax);
|
||||
__m512i bxs = _mm512_loadu_si512((const __m512i *) bx);
|
||||
|
||||
abx = _mm512_add_epi64(abx, _mm512_popcnt_epi64(_mm512_and_si512(axs, bxs)));
|
||||
aax = _mm512_add_epi64(aax, _mm512_popcnt_epi64(axs));
|
||||
bbx = _mm512_add_epi64(bbx, _mm512_popcnt_epi64(bxs));
|
||||
|
||||
ax += sizeof(__m512i);
|
||||
bx += sizeof(__m512i);
|
||||
}
|
||||
|
||||
ab += _mm512_reduce_add_epi64(abx);
|
||||
aa += _mm512_reduce_add_epi64(aax);
|
||||
bb += _mm512_reduce_add_epi64(bbx);
|
||||
|
||||
return BitJaccardDistanceDefault(bytes, ax, bx, ab, aa, bb);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef BIT_DISPATCH
|
||||
#define CPU_FEATURE_OSXSAVE (1 << 27) /* F1 ECX */
|
||||
#define CPU_FEATURE_AVX512F (1 << 16) /* F7,0 EBX */
|
||||
#define CPU_FEATURE_AVX512VPOPCNTDQ (1 << 14) /* F7,0 ECX */
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define TARGET_XSAVE
|
||||
#else
|
||||
#define TARGET_XSAVE __attribute__((target("xsave")))
|
||||
#endif
|
||||
|
||||
TARGET_XSAVE static bool
|
||||
SupportsAvx512Popcount()
|
||||
{
|
||||
unsigned int exx[4] = {0, 0, 0, 0};
|
||||
|
||||
#if defined(HAVE__GET_CPUID)
|
||||
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
|
||||
#else
|
||||
__cpuid(exx, 1);
|
||||
#endif
|
||||
|
||||
/* Check OS supports XSAVE */
|
||||
if ((exx[2] & CPU_FEATURE_OSXSAVE) != CPU_FEATURE_OSXSAVE)
|
||||
return false;
|
||||
|
||||
/* Check XMM, YMM, and ZMM registers are enabled */
|
||||
if ((_xgetbv(0) & 0xe6) != 0xe6)
|
||||
return false;
|
||||
|
||||
#if defined(HAVE__GET_CPUID)
|
||||
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]);
|
||||
#else
|
||||
__cpuidex(exx, 7, 0);
|
||||
#endif
|
||||
|
||||
/* Check AVX512F */
|
||||
if ((exx[1] & CPU_FEATURE_AVX512F) != CPU_FEATURE_AVX512F)
|
||||
return false;
|
||||
|
||||
/* Check AVX512VPOPCNTDQ */
|
||||
return (exx[2] & CPU_FEATURE_AVX512VPOPCNTDQ) == CPU_FEATURE_AVX512VPOPCNTDQ;
|
||||
}
|
||||
#endif
|
||||
|
||||
void
|
||||
BitvecInit(void)
|
||||
{
|
||||
/*
|
||||
* Could skip pointer when single function, but no difference in
|
||||
* performance
|
||||
*/
|
||||
BitHammingDistance = BitHammingDistanceDefault;
|
||||
BitJaccardDistance = BitJaccardDistanceDefault;
|
||||
|
||||
#ifdef BIT_DISPATCH
|
||||
if (SupportsAvx512Popcount())
|
||||
{
|
||||
BitHammingDistance = BitHammingDistanceAvx512Popcount;
|
||||
BitJaccardDistance = BitJaccardDistanceAvx512Popcount;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
#ifndef BITUTILS_H
|
||||
#define BITUTILS_H
|
||||
|
||||
/* We use two types of dispatching: intrinsics and target_clones */
|
||||
/* TODO Move to better place */
|
||||
#ifndef DISABLE_DISPATCH
|
||||
/* Only enable for more recent compilers to keep build process simple */
|
||||
#if defined(__x86_64__) && defined(__GNUC__) && __GNUC__ >= 8
|
||||
#define USE_DISPATCH
|
||||
#elif defined(__x86_64__) && defined(__clang_major__) && __clang_major__ >= 7
|
||||
#define USE_DISPATCH
|
||||
#elif defined(_M_AMD64) && defined(_MSC_VER) && _MSC_VER >= 1920
|
||||
#define USE_DISPATCH
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* target_clones requires glibc */
|
||||
#if defined(USE_DISPATCH) && defined(__gnu_linux__)
|
||||
#define USE_TARGET_CLONES
|
||||
#endif
|
||||
|
||||
#if defined(USE_DISPATCH)
|
||||
#define BIT_DISPATCH
|
||||
#endif
|
||||
|
||||
extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);
|
||||
extern double (*BitJaccardDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb);
|
||||
|
||||
void BitvecInit(void);
|
||||
|
||||
#endif
|
||||
@@ -1,7 +1,7 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include "bitutils.h"
|
||||
#include "bitvec.h"
|
||||
#include "bitvector.h"
|
||||
#include "port/pg_bitutils.h"
|
||||
#include "utils/varbit.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 160000
|
||||
@@ -46,10 +46,17 @@ hamming_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
VarBit *a = PG_GETARG_VARBIT_P(0);
|
||||
VarBit *b = PG_GETARG_VARBIT_P(1);
|
||||
unsigned char *ax = VARBITS(a);
|
||||
unsigned char *bx = VARBITS(b);
|
||||
uint64 distance = 0;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_FLOAT8((double) BitHammingDistance(VARBITBYTES(a), VARBITS(a), VARBITS(b), 0));
|
||||
/* TODO Improve performance */
|
||||
for (uint32 i = 0; i < VARBITBYTES(a); i++)
|
||||
distance += pg_number_of_ones[ax[i] ^ bx[i]];
|
||||
|
||||
PG_RETURN_FLOAT8((double) distance);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -61,8 +68,23 @@ jaccard_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
VarBit *a = PG_GETARG_VARBIT_P(0);
|
||||
VarBit *b = PG_GETARG_VARBIT_P(1);
|
||||
unsigned char *ax = VARBITS(a);
|
||||
unsigned char *bx = VARBITS(b);
|
||||
uint64 ab = 0;
|
||||
uint64 aa;
|
||||
uint64 bb;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_FLOAT8(BitJaccardDistance(VARBITBYTES(a), VARBITS(a), VARBITS(b), 0, 0, 0));
|
||||
/* TODO Improve performance */
|
||||
for (uint32 i = 0; i < VARBITBYTES(a); i++)
|
||||
ab += pg_number_of_ones[ax[i] & bx[i]];
|
||||
|
||||
if (ab == 0)
|
||||
PG_RETURN_FLOAT8(1);
|
||||
|
||||
aa = pg_popcount((char *) ax, VARBITBYTES(a));
|
||||
bb = pg_popcount((char *) bx, VARBITBYTES(b));
|
||||
|
||||
PG_RETURN_FLOAT8(1 - (ab / ((double) (aa + bb - ab))));
|
||||
}
|
||||
@@ -8,21 +8,21 @@
|
||||
|
||||
#if defined(HAVE__GET_CPUID)
|
||||
#include <cpuid.h>
|
||||
#else
|
||||
#elif defined(HAVE__CPUID)
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define TARGET_F16C
|
||||
#define TARGET_F16C_FMA
|
||||
#else
|
||||
#define TARGET_F16C __attribute__((target("avx,f16c,fma")))
|
||||
#define TARGET_F16C_FMA __attribute__((target("f16c,fma")))
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
float (*HalfvecL2SquaredDistance) (int dim, half * ax, half * bx);
|
||||
float (*HalfvecInnerProduct) (int dim, half * ax, half * bx);
|
||||
double (*HalfvecCosineSimilarity) (int dim, half * ax, half * bx);
|
||||
float (*HalfvecL1Distance) (int dim, half * ax, half * bx);
|
||||
|
||||
static float
|
||||
HalfvecL2SquaredDistanceDefault(int dim, half * ax, half * bx)
|
||||
@@ -41,8 +41,8 @@ HalfvecL2SquaredDistanceDefault(int dim, half * ax, half * bx)
|
||||
}
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
TARGET_F16C static float
|
||||
HalfvecL2SquaredDistanceF16c(int dim, half * ax, half * bx)
|
||||
TARGET_F16C_FMA static float
|
||||
HalfvecL2SquaredDistanceF16cFma(int dim, half * ax, half * bx)
|
||||
{
|
||||
float distance;
|
||||
int i;
|
||||
@@ -89,8 +89,8 @@ HalfvecInnerProductDefault(int dim, half * ax, half * bx)
|
||||
}
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
TARGET_F16C static float
|
||||
HalfvecInnerProductF16c(int dim, half * ax, half * bx)
|
||||
TARGET_F16C_FMA static float
|
||||
HalfvecInnerProductF16cFma(int dim, half * ax, half * bx)
|
||||
{
|
||||
float distance;
|
||||
int i;
|
||||
@@ -142,8 +142,8 @@ HalfvecCosineSimilarityDefault(int dim, half * ax, half * bx)
|
||||
}
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
TARGET_F16C static double
|
||||
HalfvecCosineSimilarityF16c(int dim, half * ax, half * bx)
|
||||
TARGET_F16C_FMA static double
|
||||
HalfvecCosineSimilarityF16cFma(int dim, half * ax, half * bx)
|
||||
{
|
||||
float similarity;
|
||||
float norma;
|
||||
@@ -192,51 +192,6 @@ HalfvecCosineSimilarityF16c(int dim, half * ax, half * bx)
|
||||
}
|
||||
#endif
|
||||
|
||||
static float
|
||||
HalfvecL1DistanceDefault(int dim, half * ax, half * bx)
|
||||
{
|
||||
float distance = 0.0;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < dim; i++)
|
||||
distance += fabsf(HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]));
|
||||
|
||||
return distance;
|
||||
}
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
/* Does not require FMA, but keep logic simple */
|
||||
TARGET_F16C static float
|
||||
HalfvecL1DistanceF16c(int dim, half * ax, half * bx)
|
||||
{
|
||||
float distance;
|
||||
int i;
|
||||
float s[8];
|
||||
int count = (dim / 8) * 8;
|
||||
__m256 dist = _mm256_setzero_ps();
|
||||
__m256 sign = _mm256_set1_ps(-0.0);
|
||||
|
||||
for (i = 0; i < count; i += 8)
|
||||
{
|
||||
__m128i axi = _mm_loadu_si128((__m128i *) (ax + i));
|
||||
__m128i bxi = _mm_loadu_si128((__m128i *) (bx + i));
|
||||
__m256 axs = _mm256_cvtph_ps(axi);
|
||||
__m256 bxs = _mm256_cvtph_ps(bxi);
|
||||
|
||||
dist = _mm256_add_ps(dist, _mm256_andnot_ps(sign, _mm256_sub_ps(axs, bxs)));
|
||||
}
|
||||
|
||||
_mm256_storeu_ps(s, dist);
|
||||
|
||||
distance = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
|
||||
|
||||
for (; i < dim; i++)
|
||||
distance += fabsf(HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]));
|
||||
|
||||
return distance;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
#define CPU_FEATURE_FMA (1 << 12)
|
||||
#define CPU_FEATURE_OSXSAVE (1 << 27)
|
||||
@@ -256,19 +211,16 @@ SupportsCpuFeature(unsigned int feature)
|
||||
|
||||
#if defined(HAVE__GET_CPUID)
|
||||
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
|
||||
#else
|
||||
#elif defined(HAVE__CPUID)
|
||||
__cpuid(exx, 1);
|
||||
#endif
|
||||
|
||||
/* Check OS supports XSAVE */
|
||||
if ((exx[2] & CPU_FEATURE_OSXSAVE) != CPU_FEATURE_OSXSAVE)
|
||||
return false;
|
||||
|
||||
/* Check XMM and YMM registers are enabled */
|
||||
if ((_xgetbv(0) & 6) != 6)
|
||||
return false;
|
||||
|
||||
/* Now check features */
|
||||
return (exx[2] & feature) == feature;
|
||||
}
|
||||
#endif
|
||||
@@ -283,16 +235,13 @@ HalfvecInit(void)
|
||||
HalfvecL2SquaredDistance = HalfvecL2SquaredDistanceDefault;
|
||||
HalfvecInnerProduct = HalfvecInnerProductDefault;
|
||||
HalfvecCosineSimilarity = HalfvecCosineSimilarityDefault;
|
||||
HalfvecL1Distance = HalfvecL1DistanceDefault;
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
if (SupportsCpuFeature(CPU_FEATURE_AVX | CPU_FEATURE_F16C | CPU_FEATURE_FMA))
|
||||
if (SupportsCpuFeature(CPU_FEATURE_AVX | CPU_FEATURE_FMA | CPU_FEATURE_F16C))
|
||||
{
|
||||
HalfvecL2SquaredDistance = HalfvecL2SquaredDistanceF16c;
|
||||
HalfvecInnerProduct = HalfvecInnerProductF16c;
|
||||
HalfvecCosineSimilarity = HalfvecCosineSimilarityF16c;
|
||||
/* Does not require FMA, but keep logic simple */
|
||||
HalfvecL1Distance = HalfvecL1DistanceF16c;
|
||||
HalfvecL2SquaredDistance = HalfvecL2SquaredDistanceF16cFma;
|
||||
HalfvecInnerProduct = HalfvecInnerProductF16cFma;
|
||||
HalfvecCosineSimilarity = HalfvecCosineSimilarityF16cFma;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
extern float (*HalfvecL2SquaredDistance) (int dim, half * ax, half * bx);
|
||||
extern float (*HalfvecInnerProduct) (int dim, half * ax, half * bx);
|
||||
extern double (*HalfvecCosineSimilarity) (int dim, half * ax, half * bx);
|
||||
extern float (*HalfvecL1Distance) (int dim, half * ax, half * bx);
|
||||
|
||||
void HalfvecInit(void);
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include "bitvec.h"
|
||||
#include "bitvector.h"
|
||||
#include "catalog/pg_type.h"
|
||||
#include "common/shortest_dec.h"
|
||||
#include "fmgr.h"
|
||||
@@ -11,7 +11,6 @@
|
||||
#include "lib/stringinfo.h"
|
||||
#include "libpq/pqformat.h"
|
||||
#include "port.h" /* for strtof() */
|
||||
#include "sparsevec.h"
|
||||
#include "utils/array.h"
|
||||
#include "utils/builtins.h"
|
||||
#include "utils/float.h"
|
||||
@@ -683,10 +682,17 @@ halfvec_l1_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
HalfVector *a = PG_GETARG_HALFVEC_P(0);
|
||||
HalfVector *b = PG_GETARG_HALFVEC_P(1);
|
||||
half *ax = a->x;
|
||||
half *bx = b->x;
|
||||
float distance = 0.0;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
PG_RETURN_FLOAT8((double) HalfvecL1Distance(a->dim, a->x, b->x));
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
distance += fabsf(HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]));
|
||||
|
||||
PG_RETURN_FLOAT8((double) distance);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1168,26 +1174,3 @@ halfvec_avg(PG_FUNCTION_ARGS)
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert sparse vector to half vector
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_halfvec);
|
||||
Datum
|
||||
sparsevec_to_halfvec(PG_FUNCTION_ARGS)
|
||||
{
|
||||
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
|
||||
int32 typmod = PG_GETARG_INT32(1);
|
||||
HalfVector *result;
|
||||
int dim = svec->dim;
|
||||
float *values = SPARSEVEC_VALUES(svec);
|
||||
|
||||
CheckDim(dim);
|
||||
CheckExpectedDim(typmod, dim);
|
||||
|
||||
result = InitHalfVector(dim);
|
||||
for (int i = 0; i < svec->nnz; i++)
|
||||
result->x[svec->indices[i] - 1] = Float4ToHalf(values[i]);
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
@@ -5,11 +5,10 @@
|
||||
|
||||
#include <float.h>
|
||||
|
||||
#include "bitutils.h"
|
||||
#include "fmgr.h"
|
||||
#include "vector.h"
|
||||
|
||||
#if defined(USE_DISPATCH)
|
||||
#if defined(__x86_64__) || defined(_M_AMD64)
|
||||
#define HALFVEC_DISPATCH
|
||||
#endif
|
||||
|
||||
@@ -45,5 +44,6 @@ typedef struct HalfVector
|
||||
|
||||
HalfVector *InitHalfVector(int dim);
|
||||
int halfvec_cmp_internal(HalfVector * a, HalfVector * b);
|
||||
PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -194,7 +194,7 @@ hnswhandler(PG_FUNCTION_ARGS)
|
||||
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
|
||||
|
||||
amroutine->amstrategies = 0;
|
||||
amroutine->amsupport = 5;
|
||||
amroutine->amsupport = 2;
|
||||
#if PG_VERSION_NUM >= 130000
|
||||
amroutine->amoptsprocnum = 0;
|
||||
#endif
|
||||
|
||||
20
src/hnsw.h
20
src/hnsw.h
@@ -22,9 +22,6 @@
|
||||
/* Support functions */
|
||||
#define HNSW_DISTANCE_PROC 1
|
||||
#define HNSW_NORM_PROC 2
|
||||
#define HNSW_NORMALIZE_PROC 3
|
||||
#define HNSW_MAX_DIMS_PROC 4
|
||||
#define HNSW_CHECK_VALUE_PROC 5
|
||||
|
||||
#define HNSW_VERSION 1
|
||||
#define HNSW_MAGIC_NUMBER 0xA953A953
|
||||
@@ -59,6 +56,14 @@
|
||||
#define HNSW_UPDATE_ENTRY_GREATER 1
|
||||
#define HNSW_UPDATE_ENTRY_ALWAYS 2
|
||||
|
||||
typedef enum HnswType
|
||||
{
|
||||
HNSW_TYPE_VECTOR,
|
||||
HNSW_TYPE_HALFVEC,
|
||||
HNSW_TYPE_BIT,
|
||||
HNSW_TYPE_SPARSEVEC
|
||||
} HnswType;
|
||||
|
||||
/* Build phases */
|
||||
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
|
||||
#define PROGRESS_HNSW_PHASE_LOAD 2
|
||||
@@ -246,6 +251,7 @@ typedef struct HnswBuildState
|
||||
Relation index;
|
||||
IndexInfo *indexInfo;
|
||||
ForkNumber forkNum;
|
||||
HnswType type;
|
||||
|
||||
/* Settings */
|
||||
int dimensions;
|
||||
@@ -259,8 +265,6 @@ typedef struct HnswBuildState
|
||||
/* Support functions */
|
||||
FmgrInfo *procinfo;
|
||||
FmgrInfo *normprocinfo;
|
||||
FmgrInfo *normalizeprocinfo;
|
||||
FmgrInfo *checkvalueprocinfo;
|
||||
Oid collation;
|
||||
|
||||
/* Variables */
|
||||
@@ -337,7 +341,6 @@ typedef struct HnswScanOpaqueData
|
||||
/* Support functions */
|
||||
FmgrInfo *procinfo;
|
||||
FmgrInfo *normprocinfo;
|
||||
FmgrInfo *normalizeprocinfo;
|
||||
Oid collation;
|
||||
} HnswScanOpaqueData;
|
||||
|
||||
@@ -373,9 +376,10 @@ typedef struct HnswVacuumState
|
||||
int HnswGetM(Relation index);
|
||||
int HnswGetEfConstruction(Relation index);
|
||||
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
|
||||
Datum HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum value);
|
||||
HnswType HnswGetType(Relation index);
|
||||
Datum HnswNormValue(Datum value, HnswType type);
|
||||
bool HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
|
||||
void HnswCheckValue(FmgrInfo *procinfo, Oid collation, Datum value);
|
||||
void HnswCheckValue(Datum value, HnswType type);
|
||||
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
||||
void HnswInitPage(Buffer buf, Page page);
|
||||
void HnswInit(void);
|
||||
|
||||
@@ -488,8 +488,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
||||
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
|
||||
|
||||
/* Check value */
|
||||
if (buildstate->checkvalueprocinfo != NULL)
|
||||
HnswCheckValue(buildstate->checkvalueprocinfo, buildstate->collation, value);
|
||||
HnswCheckValue(value, buildstate->type);
|
||||
|
||||
/* Normalize if needed */
|
||||
if (buildstate->normprocinfo != NULL)
|
||||
@@ -497,7 +496,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
||||
if (!HnswCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
|
||||
return false;
|
||||
|
||||
value = HnswNormValue(buildstate->normalizeprocinfo, buildstate->collation, value);
|
||||
value = HnswNormValue(value, buildstate->type);
|
||||
}
|
||||
|
||||
/* Get datum size */
|
||||
@@ -676,14 +675,18 @@ HnswSharedMemoryAlloc(Size size, void *state)
|
||||
* Get max dimensions
|
||||
*/
|
||||
static int
|
||||
GetMaxDimensions(Relation index)
|
||||
GetMaxDimensions(HnswType type)
|
||||
{
|
||||
FmgrInfo *procinfo = HnswOptionalProcInfo(index, HNSW_MAX_DIMS_PROC);
|
||||
int maxDimensions = HNSW_MAX_DIM;
|
||||
|
||||
if (procinfo == NULL)
|
||||
return HNSW_MAX_DIM;
|
||||
if (type == HNSW_TYPE_HALFVEC)
|
||||
maxDimensions *= 2;
|
||||
else if (type == HNSW_TYPE_BIT)
|
||||
maxDimensions *= 32;
|
||||
else if (type == HNSW_TYPE_SPARSEVEC)
|
||||
maxDimensions = INT_MAX;
|
||||
|
||||
return DatumGetInt32(FunctionCall1(procinfo, PointerGetDatum(NULL)));
|
||||
return maxDimensions;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -698,15 +701,13 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
||||
buildstate->index = index;
|
||||
buildstate->indexInfo = indexInfo;
|
||||
buildstate->forkNum = forkNum;
|
||||
buildstate->type = HnswGetType(index);
|
||||
|
||||
buildstate->m = HnswGetM(index);
|
||||
buildstate->efConstruction = HnswGetEfConstruction(index);
|
||||
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
|
||||
|
||||
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
|
||||
elog(ERROR, "type not supported for hnsw index");
|
||||
|
||||
maxDimensions = GetMaxDimensions(index);
|
||||
maxDimensions = GetMaxDimensions(buildstate->type);
|
||||
|
||||
/* Require column to have dimensions to be indexed */
|
||||
if (buildstate->dimensions < 0)
|
||||
@@ -724,8 +725,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
||||
/* Get support functions */
|
||||
buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
|
||||
buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
|
||||
buildstate->normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
|
||||
buildstate->checkvalueprocinfo = HnswOptionalProcInfo(index, HNSW_CHECK_VALUE_PROC);
|
||||
buildstate->collation = index->rd_indcollation[0];
|
||||
|
||||
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * 1024L);
|
||||
|
||||
@@ -612,16 +612,15 @@ static void
|
||||
HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
|
||||
{
|
||||
Datum value;
|
||||
FmgrInfo *checkvalueprocinfo = HnswOptionalProcInfo(index, HNSW_CHECK_VALUE_PROC);
|
||||
FmgrInfo *normprocinfo;
|
||||
Oid collation = index->rd_indcollation[0];
|
||||
HnswType type = HnswGetType(index);
|
||||
|
||||
/* Detoast once for all calls */
|
||||
value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
|
||||
|
||||
/* Check value */
|
||||
if (checkvalueprocinfo != NULL)
|
||||
HnswCheckValue(checkvalueprocinfo, collation, value);
|
||||
HnswCheckValue(value, type);
|
||||
|
||||
/* Normalize if needed */
|
||||
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
|
||||
@@ -630,7 +629,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
|
||||
if (!HnswCheckNorm(normprocinfo, collation, value))
|
||||
return;
|
||||
|
||||
value = HnswNormValue(HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC), collation, value);
|
||||
value = HnswNormValue(value, type);
|
||||
}
|
||||
|
||||
HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false);
|
||||
|
||||
@@ -61,7 +61,7 @@ GetScanValue(IndexScanDesc scan)
|
||||
|
||||
/* Fine if normalization fails */
|
||||
if (so->normprocinfo != NULL)
|
||||
value = HnswNormValue(so->normalizeprocinfo, so->collation, value);
|
||||
value = HnswNormValue(value, HnswGetType(scan->indexRelation));
|
||||
}
|
||||
|
||||
return value;
|
||||
@@ -87,7 +87,6 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
|
||||
/* Set support functions */
|
||||
so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
|
||||
so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
|
||||
so->normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
|
||||
so->collation = index->rd_indcollation[0];
|
||||
|
||||
scan->opaque = so;
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
#include "catalog/pg_type.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
#include "fmgr.h"
|
||||
#include "halfutils.h"
|
||||
#include "halfvec.h"
|
||||
#include "hnsw.h"
|
||||
#include "lib/pairingheap.h"
|
||||
#include "sparsevec.h"
|
||||
@@ -13,6 +15,8 @@
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memdebug.h"
|
||||
#include "utils/rel.h"
|
||||
#include "utils/syscache.h"
|
||||
#include "vector.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 130000
|
||||
#include "common/hashfn.h"
|
||||
@@ -152,16 +156,57 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
|
||||
return index_getprocinfo(index, 1, procnum);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get type
|
||||
*/
|
||||
HnswType
|
||||
HnswGetType(Relation index)
|
||||
{
|
||||
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
|
||||
HeapTuple tuple;
|
||||
Form_pg_type type;
|
||||
HnswType result;
|
||||
|
||||
if (typid == BITOID)
|
||||
return HNSW_TYPE_BIT;
|
||||
|
||||
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid));
|
||||
if (!HeapTupleIsValid(tuple))
|
||||
elog(ERROR, "cache lookup failed for type %u", typid);
|
||||
|
||||
type = (Form_pg_type) GETSTRUCT(tuple);
|
||||
if (strcmp(NameStr(type->typname), "vector") == 0)
|
||||
result = HNSW_TYPE_VECTOR;
|
||||
else if (strcmp(NameStr(type->typname), "halfvec") == 0)
|
||||
result = HNSW_TYPE_HALFVEC;
|
||||
else if (strcmp(NameStr(type->typname), "sparsevec") == 0)
|
||||
result = HNSW_TYPE_SPARSEVEC;
|
||||
else
|
||||
{
|
||||
ReleaseSysCache(tuple);
|
||||
elog(ERROR, "type not supported for hnsw index");
|
||||
}
|
||||
|
||||
ReleaseSysCache(tuple);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Normalize value
|
||||
*/
|
||||
Datum
|
||||
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum value)
|
||||
HnswNormValue(Datum value, HnswType type)
|
||||
{
|
||||
if (procinfo == NULL)
|
||||
/* TODO Remove type-specific code */
|
||||
if (type == HNSW_TYPE_VECTOR)
|
||||
return DirectFunctionCall1(l2_normalize, value);
|
||||
|
||||
return FunctionCall1Coll(procinfo, collation, value);
|
||||
else if (type == HNSW_TYPE_HALFVEC)
|
||||
return DirectFunctionCall1(halfvec_l2_normalize, value);
|
||||
else if (type == HNSW_TYPE_SPARSEVEC)
|
||||
return DirectFunctionCall1(sparsevec_l2_normalize, value);
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -177,9 +222,15 @@ HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
|
||||
* Check if a value can be indexed
|
||||
*/
|
||||
void
|
||||
HnswCheckValue(FmgrInfo *procinfo, Oid collation, Datum value)
|
||||
HnswCheckValue(Datum value, HnswType type)
|
||||
{
|
||||
FunctionCall1Coll(procinfo, collation, value);
|
||||
if (type == HNSW_TYPE_SPARSEVEC)
|
||||
{
|
||||
SparseVector *vec = DatumGetSparseVector(value);
|
||||
|
||||
if (vec->nnz > HNSW_MAX_NNZ)
|
||||
elog(ERROR, "sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1275,36 +1326,3 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
ep = w;
|
||||
}
|
||||
}
|
||||
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_hnsw_max_dims);
|
||||
Datum
|
||||
halfvec_hnsw_max_dims(PG_FUNCTION_ARGS)
|
||||
{
|
||||
PG_RETURN_INT32(HNSW_MAX_DIM * 2);
|
||||
};
|
||||
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(bit_hnsw_max_dims);
|
||||
Datum
|
||||
bit_hnsw_max_dims(PG_FUNCTION_ARGS)
|
||||
{
|
||||
PG_RETURN_INT32(HNSW_MAX_DIM * 32);
|
||||
};
|
||||
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_hnsw_max_dims);
|
||||
Datum
|
||||
sparsevec_hnsw_max_dims(PG_FUNCTION_ARGS)
|
||||
{
|
||||
PG_RETURN_INT32(SPARSEVEC_MAX_DIM);
|
||||
};
|
||||
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_hnsw_check_value);
|
||||
Datum
|
||||
sparsevec_hnsw_check_value(PG_FUNCTION_ARGS)
|
||||
{
|
||||
SparseVector *vec = PG_GETARG_SPARSEVEC_P(0);
|
||||
|
||||
if (vec->nnz > HNSW_MAX_NNZ)
|
||||
elog(ERROR, "sparsevec cannot have more than %d non-zero elements for hnsw index", HNSW_MAX_NNZ);
|
||||
|
||||
PG_RETURN_VOID();
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
#include "access/tableam.h"
|
||||
#include "access/parallel.h"
|
||||
#include "access/xact.h"
|
||||
#include "bitvec.h"
|
||||
#include "bitvector.h"
|
||||
#include "catalog/index.h"
|
||||
#include "catalog/pg_operator_d.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
@@ -60,10 +60,8 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
|
||||
*/
|
||||
if (buildstate->kmeansnormprocinfo != NULL)
|
||||
{
|
||||
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
|
||||
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value, buildstate->type))
|
||||
return;
|
||||
|
||||
value = IvfflatNormValue(buildstate->normalizeprocinfo, buildstate->collation, value);
|
||||
}
|
||||
|
||||
if (samples->length < targsamples)
|
||||
@@ -158,10 +156,8 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
|
||||
/* Normalize if needed */
|
||||
if (buildstate->normprocinfo != NULL)
|
||||
{
|
||||
if (!IvfflatCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
|
||||
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->type))
|
||||
return;
|
||||
|
||||
value = IvfflatNormValue(buildstate->normalizeprocinfo, buildstate->collation, value);
|
||||
}
|
||||
|
||||
/* Find the list that minimizes the distance */
|
||||
@@ -383,7 +379,6 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
|
||||
buildstate->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
|
||||
buildstate->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
||||
buildstate->kmeansnormprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
|
||||
buildstate->normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
|
||||
buildstate->collation = index->rd_indcollation[0];
|
||||
|
||||
/* Require more than one dimension for spherical k-means */
|
||||
|
||||
@@ -188,7 +188,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
|
||||
|
||||
amroutine->amstrategies = 0;
|
||||
amroutine->amsupport = 6;
|
||||
amroutine->amsupport = 4;
|
||||
#if PG_VERSION_NUM >= 130000
|
||||
amroutine->amoptsprocnum = 0;
|
||||
#endif
|
||||
|
||||
@@ -28,8 +28,6 @@
|
||||
#define IVFFLAT_NORM_PROC 2
|
||||
#define IVFFLAT_KMEANS_DISTANCE_PROC 3
|
||||
#define IVFFLAT_KMEANS_NORM_PROC 4
|
||||
#define IVFFLAT_NORMALIZE_PROC 5
|
||||
#define IVFFLAT_TYPE_SUPPORT_PROC 6
|
||||
|
||||
#define IVFFLAT_VERSION 1
|
||||
#define IVFFLAT_MAGIC_NUMBER 0x14FF1A7
|
||||
@@ -49,8 +47,7 @@ typedef enum IvfflatType
|
||||
{
|
||||
IVFFLAT_TYPE_VECTOR,
|
||||
IVFFLAT_TYPE_HALFVEC,
|
||||
IVFFLAT_TYPE_BIT,
|
||||
IVFFLAT_TYPE_UNSUPPORTED
|
||||
IVFFLAT_TYPE_BIT
|
||||
} IvfflatType;
|
||||
|
||||
/* Build phases */
|
||||
@@ -178,7 +175,6 @@ typedef struct IvfflatBuildState
|
||||
FmgrInfo *procinfo;
|
||||
FmgrInfo *normprocinfo;
|
||||
FmgrInfo *kmeansnormprocinfo;
|
||||
FmgrInfo *normalizeprocinfo;
|
||||
Oid collation;
|
||||
|
||||
/* Variables */
|
||||
@@ -259,9 +255,7 @@ typedef struct IvfflatScanOpaqueData
|
||||
/* Support functions */
|
||||
FmgrInfo *procinfo;
|
||||
FmgrInfo *normprocinfo;
|
||||
FmgrInfo *normalizeprocinfo;
|
||||
Oid collation;
|
||||
Datum (*distfunc) (FmgrInfo *flinfo, Oid collation, Datum arg1, Datum arg2);
|
||||
|
||||
/* Lists */
|
||||
pairingheap *listQueue;
|
||||
@@ -281,8 +275,7 @@ void VectorArrayFree(VectorArray arr);
|
||||
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, IvfflatType type);
|
||||
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
|
||||
IvfflatType IvfflatGetType(Relation index);
|
||||
Datum IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum value);
|
||||
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
|
||||
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, IvfflatType type);
|
||||
int IvfflatGetLists(Relation index);
|
||||
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
|
||||
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
|
||||
|
||||
@@ -85,12 +85,8 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
|
||||
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
||||
if (normprocinfo != NULL)
|
||||
{
|
||||
Oid collation = index->rd_indcollation[0];
|
||||
|
||||
if (!IvfflatCheckNorm(normprocinfo, collation, value))
|
||||
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value, IvfflatGetType(index)))
|
||||
return;
|
||||
|
||||
value = IvfflatNormValue(IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC), collation, value);
|
||||
}
|
||||
|
||||
/* Find the insert page - sets the page and list info */
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "bitvec.h"
|
||||
#include "bitvector.h"
|
||||
#include "halfutils.h"
|
||||
#include "halfvec.h"
|
||||
#include "ivfflat.h"
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
#include <float.h>
|
||||
|
||||
#include "access/relscan.h"
|
||||
#include "bitvector.h"
|
||||
#include "catalog/pg_operator_d.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
#include "halfvec.h"
|
||||
#include "lib/pairingheap.h"
|
||||
#include "ivfflat.h"
|
||||
#include "miscadmin.h"
|
||||
@@ -56,7 +58,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
|
||||
double distance;
|
||||
|
||||
/* Use procinfo from the index instead of scan key for performance */
|
||||
distance = DatumGetFloat8(so->distfunc(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
|
||||
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
|
||||
|
||||
if (listCount < so->probes)
|
||||
{
|
||||
@@ -149,7 +151,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
* performance
|
||||
*/
|
||||
ExecClearTuple(slot);
|
||||
slot->tts_values[0] = so->distfunc(so->procinfo, so->collation, datum, value);
|
||||
slot->tts_values[0] = FunctionCall2Coll(so->procinfo, so->collation, datum, value);
|
||||
slot->tts_isnull[0] = false;
|
||||
slot->tts_values[1] = PointerGetDatum(&itup->t_tid);
|
||||
slot->tts_isnull[1] = false;
|
||||
@@ -177,15 +179,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
tuplesort_performsort(so->sortstate);
|
||||
}
|
||||
|
||||
/*
|
||||
* Zero distance
|
||||
*/
|
||||
static Datum
|
||||
ZeroDistance(FmgrInfo *flinfo, Oid collation, Datum arg1, Datum arg2)
|
||||
{
|
||||
return Float8GetDatum(0.0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get scan value
|
||||
*/
|
||||
@@ -197,21 +190,28 @@ GetScanValue(IndexScanDesc scan)
|
||||
|
||||
if (scan->orderByData->sk_flags & SK_ISNULL)
|
||||
{
|
||||
value = PointerGetDatum(NULL);
|
||||
so->distfunc = ZeroDistance;
|
||||
IvfflatType type = IvfflatGetType(scan->indexRelation);
|
||||
|
||||
if (type == IVFFLAT_TYPE_VECTOR)
|
||||
value = PointerGetDatum(InitVector(so->dimensions));
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
value = PointerGetDatum(InitHalfVector(so->dimensions));
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
value = PointerGetDatum(InitBitVector(so->dimensions));
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
}
|
||||
else
|
||||
{
|
||||
value = scan->orderByData->sk_argument;
|
||||
so->distfunc = FunctionCall2Coll;
|
||||
|
||||
/* Value should not be compressed or toasted */
|
||||
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
|
||||
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
|
||||
|
||||
/* Check normprocinfo since normalizeprocinfo not set for vector */
|
||||
/* Fine if normalization fails */
|
||||
if (so->normprocinfo != NULL)
|
||||
value = IvfflatNormValue(so->normalizeprocinfo, so->collation, value);
|
||||
IvfflatNormValue(so->normprocinfo, so->collation, &value, IvfflatGetType(scan->indexRelation));
|
||||
}
|
||||
|
||||
return value;
|
||||
@@ -249,7 +249,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
|
||||
/* Set support functions */
|
||||
so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
|
||||
so->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
||||
so->normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
|
||||
so->collation = index->rd_indcollation[0];
|
||||
|
||||
/* Create tuple description for sorting */
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
#include "access/generic_xlog.h"
|
||||
#include "catalog/pg_type.h"
|
||||
#include "fmgr.h"
|
||||
#include "halfutils.h"
|
||||
#include "halfvec.h"
|
||||
#include "ivfflat.h"
|
||||
#include "storage/bufmgr.h"
|
||||
#include "vector.h"
|
||||
#include "utils/syscache.h"
|
||||
|
||||
/*
|
||||
* Allocate a vector array
|
||||
@@ -67,40 +71,60 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
|
||||
IvfflatType
|
||||
IvfflatGetType(Relation index)
|
||||
{
|
||||
FmgrInfo *procinfo = IvfflatOptionalProcInfo(index, IVFFLAT_TYPE_SUPPORT_PROC);
|
||||
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
|
||||
HeapTuple tuple;
|
||||
Form_pg_type type;
|
||||
IvfflatType result;
|
||||
|
||||
if (procinfo == NULL)
|
||||
return IVFFLAT_TYPE_VECTOR;
|
||||
if (typid == BITOID)
|
||||
return IVFFLAT_TYPE_BIT;
|
||||
|
||||
result = (IvfflatType) DatumGetInt32(FunctionCall1(procinfo, ObjectIdGetDatum(typid)));
|
||||
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid));
|
||||
if (!HeapTupleIsValid(tuple))
|
||||
elog(ERROR, "cache lookup failed for type %u", typid);
|
||||
|
||||
if (result == IVFFLAT_TYPE_UNSUPPORTED)
|
||||
type = (Form_pg_type) GETSTRUCT(tuple);
|
||||
if (strcmp(NameStr(type->typname), "vector") == 0)
|
||||
result = IVFFLAT_TYPE_VECTOR;
|
||||
else if (strcmp(NameStr(type->typname), "halfvec") == 0)
|
||||
result = IVFFLAT_TYPE_HALFVEC;
|
||||
else
|
||||
{
|
||||
ReleaseSysCache(tuple);
|
||||
elog(ERROR, "type not supported for ivfflat index");
|
||||
}
|
||||
|
||||
ReleaseSysCache(tuple);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Normalize value
|
||||
*/
|
||||
Datum
|
||||
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum value)
|
||||
{
|
||||
if (procinfo == NULL)
|
||||
return DirectFunctionCall1(l2_normalize, value);
|
||||
|
||||
return FunctionCall1Coll(procinfo, collation, value);
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if non-zero norm
|
||||
* Divide by the norm
|
||||
*
|
||||
* Returns false if value should not be indexed
|
||||
*
|
||||
* The caller needs to free the pointer stored in value
|
||||
* if it's different than the original value
|
||||
*/
|
||||
bool
|
||||
IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
|
||||
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, IvfflatType type)
|
||||
{
|
||||
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
|
||||
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
|
||||
|
||||
if (norm > 0)
|
||||
{
|
||||
if (type == IVFFLAT_TYPE_VECTOR)
|
||||
*value = DirectFunctionCall1(l2_normalize, *value);
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
*value = DirectFunctionCall1(halfvec_l2_normalize, *value);
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -245,22 +269,3 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
}
|
||||
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ivfflat_support);
|
||||
Datum
|
||||
halfvec_ivfflat_support(PG_FUNCTION_ARGS)
|
||||
{
|
||||
PG_RETURN_INT32(IVFFLAT_TYPE_HALFVEC);
|
||||
};
|
||||
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(bit_ivfflat_support);
|
||||
Datum
|
||||
bit_ivfflat_support(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Oid typid = PG_GETARG_OID(0);
|
||||
|
||||
if (typid == BITOID)
|
||||
PG_RETURN_INT32(IVFFLAT_TYPE_BIT);
|
||||
else
|
||||
PG_RETURN_INT32(IVFFLAT_TYPE_UNSUPPORTED);
|
||||
};
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
#include <math.h>
|
||||
|
||||
#include "fmgr.h"
|
||||
#include "halfutils.h"
|
||||
#include "halfvec.h"
|
||||
#include "libpq/pqformat.h"
|
||||
#include "sparsevec.h"
|
||||
#include "utils/array.h"
|
||||
@@ -20,12 +18,6 @@
|
||||
#include "utils/builtins.h"
|
||||
#endif
|
||||
|
||||
typedef struct SparseInputElement
|
||||
{
|
||||
int32 index;
|
||||
float value;
|
||||
} SparseInputElement;
|
||||
|
||||
/*
|
||||
* Ensure same dimensions
|
||||
*/
|
||||
@@ -172,21 +164,6 @@ sparsevec_isspace(char ch)
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compare indices
|
||||
*/
|
||||
static int
|
||||
CompareIndices(const void *a, const void *b)
|
||||
{
|
||||
if (((SparseInputElement *) a)->index < ((SparseInputElement *) b)->index)
|
||||
return -1;
|
||||
|
||||
if (((SparseInputElement *) a)->index > ((SparseInputElement *) b)->index)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert textual representation to internal representation
|
||||
*/
|
||||
@@ -201,7 +178,8 @@ sparsevec_in(PG_FUNCTION_ARGS)
|
||||
char *stringEnd;
|
||||
SparseVector *result;
|
||||
float *rvalues;
|
||||
SparseInputElement *elements;
|
||||
int32 *indices;
|
||||
float *values;
|
||||
int maxNnz;
|
||||
int nnz = 0;
|
||||
|
||||
@@ -219,7 +197,8 @@ sparsevec_in(PG_FUNCTION_ARGS)
|
||||
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
||||
errmsg("sparsevec cannot have more than %d non-zero elements", SPARSEVEC_MAX_NNZ)));
|
||||
|
||||
elements = palloc(maxNnz * sizeof(SparseInputElement));
|
||||
indices = palloc(maxNnz * sizeof(int32));
|
||||
values = palloc(maxNnz * sizeof(float));
|
||||
|
||||
pt = lit;
|
||||
|
||||
@@ -312,8 +291,8 @@ sparsevec_in(PG_FUNCTION_ARGS)
|
||||
/* Do not store zero values */
|
||||
if (value != 0)
|
||||
{
|
||||
elements[nnz].index = index;
|
||||
elements[nnz].value = value;
|
||||
indices[nnz] = index;
|
||||
values[nnz] = value;
|
||||
nnz++;
|
||||
}
|
||||
|
||||
@@ -374,14 +353,12 @@ sparsevec_in(PG_FUNCTION_ARGS)
|
||||
CheckDim(dim);
|
||||
CheckExpectedDim(typmod, dim);
|
||||
|
||||
qsort(elements, nnz, sizeof(SparseInputElement), CompareIndices);
|
||||
|
||||
result = InitSparseVector(dim, nnz);
|
||||
rvalues = SPARSEVEC_VALUES(result);
|
||||
for (int i = 0; i < nnz; i++)
|
||||
{
|
||||
result->indices[i] = elements[i].index;
|
||||
rvalues[i] = elements[i].value;
|
||||
result->indices[i] = indices[i];
|
||||
rvalues[i] = values[i];
|
||||
|
||||
CheckIndex(result->indices, i, dim);
|
||||
}
|
||||
@@ -527,10 +504,6 @@ sparsevec_recv(PG_FUNCTION_ARGS)
|
||||
{
|
||||
values[i] = pq_getmsgfloat4(buf);
|
||||
CheckElement(values[i]);
|
||||
if (values[i] == 0)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_DATA_EXCEPTION),
|
||||
errmsg("binary representation of sparsevec cannot contain zero values")));
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
@@ -618,49 +591,6 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert half vector to sparse vector
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_sparsevec);
|
||||
Datum
|
||||
halfvec_to_sparsevec(PG_FUNCTION_ARGS)
|
||||
{
|
||||
HalfVector *vec = PG_GETARG_HALFVEC_P(0);
|
||||
int32 typmod = PG_GETARG_INT32(1);
|
||||
SparseVector *result;
|
||||
int dim = vec->dim;
|
||||
int nnz = 0;
|
||||
float *values;
|
||||
int j = 0;
|
||||
|
||||
CheckDim(dim);
|
||||
CheckExpectedDim(typmod, dim);
|
||||
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
if (!HalfIsZero(vec->x[i]))
|
||||
nnz++;
|
||||
}
|
||||
|
||||
result = InitSparseVector(dim, nnz);
|
||||
values = SPARSEVEC_VALUES(result);
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
if (!HalfIsZero(vec->x[i]))
|
||||
{
|
||||
/* Safety check */
|
||||
if (j >= result->nnz)
|
||||
elog(ERROR, "safety check failed");
|
||||
|
||||
result->indices[j] = i + 1;
|
||||
values[j] = HalfToFloat4(vec->x[i]);
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the L2 squared distance between sparse vectors
|
||||
*/
|
||||
|
||||
@@ -23,5 +23,6 @@ typedef struct SparseVector
|
||||
} SparseVector;
|
||||
|
||||
SparseVector *InitSparseVector(int dim, int nnz);
|
||||
PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
|
||||
#endif
|
||||
|
||||
27
src/vector.c
27
src/vector.c
@@ -2,8 +2,7 @@
|
||||
|
||||
#include <math.h>
|
||||
|
||||
#include "bitutils.h"
|
||||
#include "bitvec.h"
|
||||
#include "bitvector.h"
|
||||
#include "catalog/pg_type.h"
|
||||
#include "common/shortest_dec.h"
|
||||
#include "fmgr.h"
|
||||
@@ -34,10 +33,18 @@
|
||||
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
|
||||
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
|
||||
|
||||
#if defined(USE_TARGET_CLONES) && !defined(__FMA__)
|
||||
#define VECTOR_TARGET_CLONES __attribute__((target_clones("default", "fma")))
|
||||
/* target_clones requires glibc */
|
||||
#if defined(__gnu_linux__) && defined(__has_attribute)
|
||||
/* Use separate line for portability */
|
||||
#if __has_attribute(target_clones)
|
||||
#define HAVE_TARGET_CLONES
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__x86_64__) && defined(HAVE_TARGET_CLONES) && !defined(__FMA__)
|
||||
#define VECTOR_DISPATCH __attribute__((target_clones("default", "fma")))
|
||||
#else
|
||||
#define VECTOR_TARGET_CLONES
|
||||
#define VECTOR_DISPATCH
|
||||
#endif
|
||||
|
||||
PG_MODULE_MAGIC;
|
||||
@@ -49,7 +56,6 @@ PGDLLEXPORT void _PG_init(void);
|
||||
void
|
||||
_PG_init(void)
|
||||
{
|
||||
BitvecInit();
|
||||
HalfvecInit();
|
||||
HnswInit();
|
||||
IvfflatInit();
|
||||
@@ -565,7 +571,7 @@ halfvec_to_vector(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
VECTOR_TARGET_CLONES static float
|
||||
VECTOR_DISPATCH static float
|
||||
VectorL2SquaredDistance(int dim, float *ax, float *bx)
|
||||
{
|
||||
float distance = 0.0;
|
||||
@@ -612,7 +618,7 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_FLOAT8((double) VectorL2SquaredDistance(a->dim, a->x, b->x));
|
||||
}
|
||||
|
||||
VECTOR_TARGET_CLONES static float
|
||||
VECTOR_DISPATCH static float
|
||||
VectorInnerProduct(int dim, float *ax, float *bx)
|
||||
{
|
||||
float distance = 0.0;
|
||||
@@ -654,7 +660,7 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_FLOAT8((double) -VectorInnerProduct(a->dim, a->x, b->x));
|
||||
}
|
||||
|
||||
VECTOR_TARGET_CLONES static double
|
||||
VECTOR_DISPATCH static double
|
||||
VectorCosineSimilarity(int dim, float *ax, float *bx)
|
||||
{
|
||||
float similarity = 0.0;
|
||||
@@ -729,8 +735,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_FLOAT8(acos(distance) / M_PI);
|
||||
}
|
||||
|
||||
/* Does not require FMA, but keep logic simple */
|
||||
VECTOR_TARGET_CLONES static float
|
||||
static float
|
||||
VectorL1Distance(int dim, float *ax, float *bx)
|
||||
{
|
||||
float distance = 0.0;
|
||||
|
||||
@@ -28,24 +28,6 @@ SELECT hamming_distance('10101010101010101010', '01010101010101010101');
|
||||
20
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
|
||||
hamming_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
|
||||
hamming_distance
|
||||
------------------
|
||||
513
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
|
||||
hamming_distance
|
||||
------------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT hamming_distance('', '');
|
||||
hamming_distance
|
||||
------------------
|
||||
@@ -104,24 +86,6 @@ SELECT jaccard_distance('10101010101010101010', '01010101010101010101');
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
|
||||
jaccard_distance
|
||||
------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
|
||||
jaccard_distance
|
||||
------------------
|
||||
0.5
|
||||
(1 row)
|
||||
|
||||
SELECT jaccard_distance('', '');
|
||||
jaccard_distance
|
||||
------------------
|
||||
|
||||
@@ -170,44 +170,6 @@ SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(4);
|
||||
ERROR: expected 4 dimensions, not 5
|
||||
SELECT '{}/16001'::sparsevec::vector;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec;
|
||||
sparsevec
|
||||
-----------------
|
||||
{2:1.5,4:3.5}/5
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(5);
|
||||
sparsevec
|
||||
-----------------
|
||||
{2:1.5,4:3.5}/5
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(4);
|
||||
ERROR: expected 4 dimensions, not 5
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec;
|
||||
halfvec
|
||||
-----------------
|
||||
[0,1.5,0,3.5,0]
|
||||
(1 row)
|
||||
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(5);
|
||||
halfvec
|
||||
-----------------
|
||||
[0,1.5,0,3.5,0]
|
||||
(1 row)
|
||||
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(4);
|
||||
ERROR: expected 4 dimensions, not 5
|
||||
SELECT '{}/16001'::sparsevec::halfvec;
|
||||
ERROR: halfvec cannot have more than 16000 dimensions
|
||||
SELECT '{1:65520}/1'::sparsevec::halfvec;
|
||||
ERROR: "65520" is out of range for type halfvec
|
||||
SELECT '{1:1e-8}/1'::sparsevec::halfvec;
|
||||
halfvec
|
||||
---------
|
||||
[0]
|
||||
(1 row)
|
||||
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
|
||||
@@ -304,24 +304,6 @@ SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
|
||||
|
||||
SELECT l1_distance('[1,2]'::halfvec, '[3]');
|
||||
ERROR: different halfvec dimensions 2 and 1
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
|
||||
l1_distance
|
||||
-------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[0,3,2,5,4,7,6,9,8]');
|
||||
l1_distance
|
||||
-------------
|
||||
9
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,0]'::halfvec <+> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
7
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[3,4]'::halfvec);
|
||||
l2_normalize
|
||||
------------------------
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
[1,1,1]
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,21 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val sparsevec(3));
|
||||
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val sparsevec_l1_ops);
|
||||
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
|
||||
SELECT * FROM t ORDER BY val <+> '{1:3,2:3,3:3}/3';
|
||||
val
|
||||
-----------------
|
||||
{1:1,2:2,3:3}/3
|
||||
{1:1,2:2,3:4}/3
|
||||
{1:1,2:1,3:1}/3
|
||||
{}/3
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::sparsevec)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,21 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val vector(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val vector_l1_ops);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
[1,1,1]
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
21
test/expected/ivfflat_bit_jaccard.out
Normal file
21
test/expected/ivfflat_bit_jaccard.out
Normal file
@@ -0,0 +1,21 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val bit(4));
|
||||
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val bit_jaccard_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES (B'1110');
|
||||
SELECT * FROM t ORDER BY val <%> B'1111';
|
||||
val
|
||||
------
|
||||
1111
|
||||
1110
|
||||
1100
|
||||
0000
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -164,18 +164,8 @@ SELECT '{1:0,2:1,3:0}/3'::sparsevec;
|
||||
(1 row)
|
||||
|
||||
SELECT '{2:1,1:1}/2'::sparsevec;
|
||||
sparsevec
|
||||
-------------
|
||||
{1:1,2:1}/2
|
||||
(1 row)
|
||||
|
||||
SELECT '{1:1,1:1}/2'::sparsevec;
|
||||
ERROR: indexes must not contain duplicates
|
||||
LINE 1: SELECT '{1:1,1:1}/2'::sparsevec;
|
||||
^
|
||||
SELECT '{1:1,2:1,1:1}/2'::sparsevec;
|
||||
ERROR: indexes must not contain duplicates
|
||||
LINE 1: SELECT '{1:1,2:1,1:1}/2'::sparsevec;
|
||||
ERROR: indexes must be in ascending order
|
||||
LINE 1: SELECT '{2:1,1:1}/2'::sparsevec;
|
||||
^
|
||||
SELECT '{}/5'::sparsevec;
|
||||
sparsevec
|
||||
|
||||
@@ -202,12 +202,6 @@ SELECT l2_distance('[3e38]'::vector, '[-3e38]');
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::vector, '[1,1,1,1,1,1,1,4,5]');
|
||||
l2_distance
|
||||
-------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,0]'::vector <-> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
@@ -228,12 +222,6 @@ SELECT inner_product('[3e38]'::vector, '[3e38]');
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::vector, '[1,2,3,4,5,6,7,8,9]');
|
||||
inner_product
|
||||
---------------
|
||||
45
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::vector <#> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
@@ -290,18 +278,6 @@ SELECT cosine_distance('[3e38]'::vector, '[3e38]');
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::vector <=> '[2,4]';
|
||||
?column?
|
||||
----------
|
||||
@@ -328,24 +304,6 @@ SELECT l1_distance('[3e38]'::vector, '[-3e38]');
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
|
||||
l1_distance
|
||||
-------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[0,3,2,5,4,7,6,9,8]');
|
||||
l1_distance
|
||||
-------------
|
||||
9
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,0]'::vector <+> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
7
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[3,4]'::vector);
|
||||
l2_normalize
|
||||
--------------
|
||||
|
||||
@@ -3,9 +3,6 @@ SELECT hamming_distance('111', '110');
|
||||
SELECT hamming_distance('111', '100');
|
||||
SELECT hamming_distance('111', '000');
|
||||
SELECT hamming_distance('10101010101010101010', '01010101010101010101');
|
||||
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
|
||||
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
|
||||
SELECT hamming_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
|
||||
SELECT hamming_distance('', '');
|
||||
SELECT hamming_distance('111', '00');
|
||||
SELECT hamming_distance('111', '000'::varbit(4));
|
||||
@@ -18,9 +15,6 @@ SELECT jaccard_distance('1111', '1000');
|
||||
SELECT jaccard_distance('1111', '0000');
|
||||
SELECT jaccard_distance('1100', '1000');
|
||||
SELECT jaccard_distance('10101010101010101010', '01010101010101010101');
|
||||
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
|
||||
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
|
||||
SELECT jaccard_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
|
||||
SELECT jaccard_distance('', '');
|
||||
SELECT jaccard_distance('1111', '000');
|
||||
SELECT jaccard_distance('1111', '0000'::varbit(5));
|
||||
|
||||
@@ -47,17 +47,6 @@ SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(5);
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(4);
|
||||
SELECT '{}/16001'::sparsevec::vector;
|
||||
|
||||
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec;
|
||||
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(5);
|
||||
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(4);
|
||||
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec;
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(5);
|
||||
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(4);
|
||||
SELECT '{}/16001'::sparsevec::halfvec;
|
||||
SELECT '{1:65520}/1'::sparsevec::halfvec;
|
||||
SELECT '{1:1e-8}/1'::sparsevec::halfvec;
|
||||
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
|
||||
|
||||
@@ -68,9 +68,6 @@ SELECT '[1,2]'::halfvec <=> '[2,4]';
|
||||
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
|
||||
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
|
||||
SELECT l1_distance('[1,2]'::halfvec, '[3]');
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[0,3,2,5,4,7,6,9,8]');
|
||||
SELECT '[0,0]'::halfvec <+> '[3,4]';
|
||||
|
||||
SELECT l2_normalize('[3,4]'::halfvec);
|
||||
SELECT l2_normalize('[3,0]'::halfvec);
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val halfvec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,12 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val sparsevec(3));
|
||||
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val sparsevec_l1_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
|
||||
|
||||
SELECT * FROM t ORDER BY val <+> '{1:3,2:3,3:3}/3';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::sparsevec)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,12 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val vector(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING hnsw (val vector_l1_ops);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
12
test/sql/ivfflat_bit_jaccard.sql
Normal file
12
test/sql/ivfflat_bit_jaccard.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val bit(4));
|
||||
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val bit_jaccard_ops) WITH (lists = 1);
|
||||
|
||||
INSERT INTO t (val) VALUES (B'1110');
|
||||
|
||||
SELECT * FROM t ORDER BY val <%> B'1111';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -34,8 +34,6 @@ SELECT '{1:1a}/1'::sparsevec;
|
||||
SELECT '{1:1,}/1'::sparsevec;
|
||||
SELECT '{1:0,2:1,3:0}/3'::sparsevec;
|
||||
SELECT '{2:1,1:1}/2'::sparsevec;
|
||||
SELECT '{1:1,1:1}/2'::sparsevec;
|
||||
SELECT '{1:1,2:1,1:1}/2'::sparsevec;
|
||||
SELECT '{}/5'::sparsevec;
|
||||
SELECT '{}/-1'::sparsevec;
|
||||
SELECT '{}/100001'::sparsevec;
|
||||
|
||||
@@ -46,13 +46,11 @@ SELECT l2_distance('[0,0]'::vector, '[3,4]');
|
||||
SELECT l2_distance('[0,0]'::vector, '[0,1]');
|
||||
SELECT l2_distance('[1,2]'::vector, '[3]');
|
||||
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
|
||||
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::vector, '[1,1,1,1,1,1,1,4,5]');
|
||||
SELECT '[0,0]'::vector <-> '[3,4]';
|
||||
|
||||
SELECT inner_product('[1,2]'::vector, '[3,4]');
|
||||
SELECT inner_product('[1,2]'::vector, '[3]');
|
||||
SELECT inner_product('[3e38]'::vector, '[3e38]');
|
||||
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::vector, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT '[1,2]'::vector <#> '[3,4]';
|
||||
|
||||
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
|
||||
@@ -64,17 +62,12 @@ SELECT cosine_distance('[1,2]'::vector, '[3]');
|
||||
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
|
||||
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
|
||||
SELECT '[1,2]'::vector <=> '[2,4]';
|
||||
|
||||
SELECT l1_distance('[0,0]'::vector, '[3,4]');
|
||||
SELECT l1_distance('[0,0]'::vector, '[0,1]');
|
||||
SELECT l1_distance('[1,2]'::vector, '[3]');
|
||||
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[0,3,2,5,4,7,6,9,8]');
|
||||
SELECT '[0,0]'::vector <+> '[3,4]';
|
||||
|
||||
SELECT l2_normalize('[3,4]'::vector);
|
||||
SELECT l2_normalize('[3,0]'::vector);
|
||||
|
||||
@@ -67,8 +67,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops", "vector_l1_ops");
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
@@ -64,8 +64,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops", "vector_l1_ops");
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
@@ -71,8 +71,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("halfvec_l2_ops", "halfvec_ip_ops", "halfvec_cosine_ops", "halfvec_l1_ops");
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("halfvec_l2_ops", "halfvec_ip_ops", "halfvec_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
@@ -67,8 +67,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("sparsevec_l2_ops", "sparsevec_ip_ops", "sparsevec_cosine_ops", "sparsevec_l1_ops");
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("sparsevec_l2_ops", "sparsevec_ip_ops", "sparsevec_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
@@ -68,8 +68,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("halfvec_l2_ops", "halfvec_ip_ops", "halfvec_cosine_ops", "halfvec_l1_ops");
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("halfvec_l2_ops", "halfvec_ip_ops", "halfvec_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
@@ -64,8 +64,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("sparsevec_l2_ops", "sparsevec_ip_ops", "sparsevec_cosine_ops", "sparsevec_l1_ops");
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("sparsevec_l2_ops", "sparsevec_ip_ops", "sparsevec_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
@@ -70,8 +70,8 @@ for (1 .. 20)
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<~>");
|
||||
my @opclasses = ("bit_hamming_ops");
|
||||
my @operators = ("<~>", "<\%>");
|
||||
my @opclasses = ("bit_hamming_ops", "bit_jaccard_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user