mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
34 Commits
minivec-e2
...
hnsw-strea
| Author | SHA1 | Date | |
|---|---|---|---|
|
c01e76f2fa | ||
|
|
ab57217f48 | ||
|
|
5f6e031ccc | ||
|
|
1a1221f905 | ||
|
|
54fa16e3e3 | ||
|
|
40c3e402c7 | ||
|
|
058248fdcc | ||
|
|
73c5145b77 | ||
|
|
ec4a23fe49 | ||
|
|
38207f5640 | ||
|
|
46de265a24 | ||
|
|
b8c27914d4 | ||
|
|
2d85af51a8 | ||
|
|
e0ad441306 | ||
|
|
5776a4d937 | ||
|
|
242a12b7d5 | ||
|
|
1370dd6e86 | ||
|
|
a100dc67e5 | ||
|
|
2df9f24aad | ||
|
|
8e979ed377 | ||
|
|
4e35c6abe3 | ||
|
|
77b3d1f2a8 | ||
|
|
ecd0738728 | ||
|
|
62ffc3641c | ||
|
|
11e4d040d9 | ||
|
|
b2fa625255 | ||
|
|
a8e699c927 | ||
|
|
91541fece6 | ||
|
|
f3de487da2 | ||
|
|
721d4b7e3f | ||
|
|
28066d8fe4 | ||
|
|
495041e43b | ||
|
|
52c385c03a | ||
|
|
80cbd32dab |
@@ -1,6 +1,8 @@
|
||||
## 0.8.0 (unreleased)
|
||||
|
||||
- Added support for iterative index scans
|
||||
- Added casts for arrays to `sparsevec`
|
||||
- Improved cost estimation
|
||||
- Reduced memory usage for HNSW index scans
|
||||
- Dropped support for Postgres 12
|
||||
|
||||
|
||||
4
Makefile
4
Makefile
@@ -4,8 +4,8 @@ EXTVERSION = 0.7.4
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*--*.sql)
|
||||
DATA_built = sql/$(EXTENSION)--$(EXTVERSION).sql
|
||||
OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/minivec.o src/ivfvacuum.o src/sparsevec.o src/vector.o
|
||||
HEADERS = src/halfvec.h src/minivec.h src/sparsevec.h src/vector.h
|
||||
OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
|
||||
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
|
||||
|
||||
TESTS = $(wildcard test/sql/*.sql)
|
||||
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
|
||||
|
||||
@@ -2,10 +2,10 @@ EXTENSION = vector
|
||||
EXTVERSION = 0.7.4
|
||||
|
||||
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
|
||||
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\minivec.obj src\sparsevec.obj src\vector.obj
|
||||
HEADERS = src\halfvec.h src\minivec.h src\sparsevec.h src\vector.h
|
||||
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
|
||||
|
||||
REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector minivec sparsevec vector_type
|
||||
REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type
|
||||
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
|
||||
|
||||
# For /arch flags
|
||||
|
||||
88
README.md
88
README.md
@@ -445,6 +445,63 @@ Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html
|
||||
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
|
||||
```
|
||||
|
||||
## Streaming Queries [unreleased]
|
||||
|
||||
*Added in 0.8.0*
|
||||
|
||||
With approximate indexes, you can end up with less results than expected due to filtering conditions in the query.
|
||||
|
||||
Starting with 0.8.0, you can enable streaming queries. If too few results from the initial index scan match the query filters, it will resume scanning until enough results are found. This can significantly improve recall (at the cost of speed).
|
||||
|
||||
```tsql
|
||||
SET hnsw.streaming = on;
|
||||
-- or
|
||||
SET ivfflat.streaming = on;
|
||||
```
|
||||
|
||||
### Streaming Options
|
||||
|
||||
Since scanning a large portion of the index is expensive, there are options to control when the scan ends.
|
||||
|
||||
#### HNSW
|
||||
|
||||
Specify the max number of additional tuples visited
|
||||
|
||||
```sql
|
||||
SET hnsw.ef_stream = 10000;
|
||||
```
|
||||
|
||||
The scan will also end if reaches `work_mem`, at which point a notice is shown
|
||||
|
||||
```text
|
||||
NOTICE: hnsw index scan exceeded work_mem after 50000 tuples
|
||||
HINT: Increase work_mem to scan more tuples.
|
||||
```
|
||||
|
||||
Adjust this with:
|
||||
|
||||
```sql
|
||||
SET work_mem = '8MB';
|
||||
```
|
||||
|
||||
#### IVFFlat
|
||||
|
||||
Specify the max number of probes
|
||||
|
||||
```sql
|
||||
SET ivfflat.max_probes = 100;
|
||||
```
|
||||
|
||||
### Streaming Order
|
||||
|
||||
With streaming queries, it’s possible for rows to be slightly out of order by distance. For strict ordering, use:
|
||||
|
||||
```sql
|
||||
WITH approx_order AS MATERIALIZED (
|
||||
SELECT *, embedding <-> '[1,2,3]' AS distance FROM items WHERE ... ORDER BY distance LIMIT 5
|
||||
) SELECT * FROM approx_order ORDER BY distance;
|
||||
```
|
||||
|
||||
## Half-Precision Vectors
|
||||
|
||||
*Added in 0.7.0*
|
||||
@@ -934,37 +991,6 @@ Function | Description | Added
|
||||
avg(halfvec) → halfvec | average | 0.7.0
|
||||
sum(halfvec) → halfvec | sum | 0.7.0
|
||||
|
||||
### Minivec Type
|
||||
|
||||
Each mini vector takes `dimensions + 8` bytes of storage. Each element is a E4M3 8-bit floating-point number, and all elements must be finite (no `NaN`). Mini vectors can have up to 16,000 dimensions.
|
||||
|
||||
### Minivec Operators
|
||||
|
||||
Operator | Description | Added
|
||||
--- | --- | ---
|
||||
\+ | element-wise addition | 0.8.0
|
||||
\- | element-wise subtraction | 0.8.0
|
||||
\* | element-wise multiplication | 0.8.0
|
||||
\|\| | concatenate | 0.8.0
|
||||
<-> | Euclidean distance | 0.8.0
|
||||
<#> | negative inner product | 0.8.0
|
||||
<=> | cosine distance | 0.8.0
|
||||
<+> | taxicab distance | 0.8.0
|
||||
|
||||
### Minivec Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
binary_quantize(minivec) → bit | binary quantize | 0.8.0
|
||||
cosine_distance(minivec, minivec) → double precision | cosine distance | 0.8.0
|
||||
inner_product(minivec, minivec) → double precision | inner product | 0.8.0
|
||||
l1_distance(minivec, minivec) → double precision | taxicab distance | 0.8.0
|
||||
l2_distance(minivec, minivec) → double precision | Euclidean distance | 0.8.0
|
||||
l2_norm(minivec) → double precision | Euclidean norm | 0.8.0
|
||||
l2_normalize(minivec) → minivec | Normalize with Euclidean norm | 0.8.0
|
||||
subvector(minivec, integer, integer) → minivec | subvector | 0.8.0
|
||||
vector_dims(minivec) → integer | number of dimensions | 0.8.0
|
||||
|
||||
### Bit Type
|
||||
|
||||
Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres docs](https://www.postgresql.org/docs/current/datatype-bit.html) for more info.
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.0'" to load this file. \quit
|
||||
|
||||
-- TODO minivec functions
|
||||
|
||||
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
|
||||
295
sql/vector.sql
295
sql/vector.sql
@@ -266,18 +266,12 @@ COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
|
||||
CREATE FUNCTION ivfflat_halfvec_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C;
|
||||
|
||||
CREATE FUNCTION ivfflat_minivec_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C;
|
||||
|
||||
CREATE FUNCTION ivfflat_bit_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C;
|
||||
|
||||
CREATE FUNCTION hnsw_halfvec_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C;
|
||||
|
||||
CREATE FUNCTION hnsw_minivec_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C;
|
||||
|
||||
CREATE FUNCTION hnsw_bit_support(internal) RETURNS internal
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C;
|
||||
|
||||
@@ -653,295 +647,6 @@ CREATE OPERATOR CLASS halfvec_l1_ops
|
||||
FUNCTION 1 l1_distance(halfvec, halfvec),
|
||||
FUNCTION 3 hnsw_halfvec_support(internal);
|
||||
|
||||
-- minivec type
|
||||
|
||||
CREATE TYPE minivec;
|
||||
|
||||
CREATE FUNCTION minivec_in(cstring, oid, integer) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_out(minivec) RETURNS cstring
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_typmod_in(cstring[]) RETURNS integer
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_recv(internal, oid, integer) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_send(minivec) RETURNS bytea
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE TYPE minivec (
|
||||
INPUT = minivec_in,
|
||||
OUTPUT = minivec_out,
|
||||
TYPMOD_IN = minivec_typmod_in,
|
||||
RECEIVE = minivec_recv,
|
||||
SEND = minivec_send,
|
||||
STORAGE = external
|
||||
);
|
||||
|
||||
-- minivec functions
|
||||
|
||||
CREATE FUNCTION l2_distance(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'minivec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION inner_product(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'minivec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION cosine_distance(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'minivec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION l1_distance(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'minivec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_dims(minivec) RETURNS integer
|
||||
AS 'MODULE_PATHNAME', 'minivec_vector_dims' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION l2_norm(minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'minivec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION l2_normalize(minivec) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME', 'minivec_l2_normalize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION binary_quantize(minivec) RETURNS bit
|
||||
AS 'MODULE_PATHNAME', 'minivec_binary_quantize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION subvector(minivec, int, int) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME', 'minivec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- minivec private functions
|
||||
|
||||
CREATE FUNCTION minivec_add(minivec, minivec) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_sub(minivec, minivec) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_mul(minivec, minivec) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_concat(minivec, minivec) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_lt(minivec, minivec) RETURNS bool
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_le(minivec, minivec) RETURNS bool
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_eq(minivec, minivec) RETURNS bool
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_ne(minivec, minivec) RETURNS bool
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_ge(minivec, minivec) RETURNS bool
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_gt(minivec, minivec) RETURNS bool
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_cmp(minivec, minivec) RETURNS int4
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_l2_squared_distance(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_negative_inner_product(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_spherical_distance(minivec, minivec) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- minivec cast functions
|
||||
|
||||
CREATE FUNCTION minivec(minivec, integer, boolean) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_to_vector(minivec, integer, boolean) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_to_minivec(vector, integer, boolean) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_minivec(integer[], integer, boolean) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_minivec(real[], integer, boolean) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_minivec(double precision[], integer, boolean) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION array_to_minivec(numeric[], integer, boolean) RETURNS minivec
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION minivec_to_float4(minivec, integer, boolean) RETURNS real[]
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- minivec casts
|
||||
|
||||
CREATE CAST (minivec AS minivec)
|
||||
WITH FUNCTION minivec(minivec, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (minivec AS vector)
|
||||
WITH FUNCTION minivec_to_vector(minivec, integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (vector AS minivec)
|
||||
WITH FUNCTION vector_to_minivec(vector, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (minivec AS real[])
|
||||
WITH FUNCTION minivec_to_float4(minivec, integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (integer[] AS minivec)
|
||||
WITH FUNCTION array_to_minivec(integer[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (real[] AS minivec)
|
||||
WITH FUNCTION array_to_minivec(real[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (double precision[] AS minivec)
|
||||
WITH FUNCTION array_to_minivec(double precision[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
CREATE CAST (numeric[] AS minivec)
|
||||
WITH FUNCTION array_to_minivec(numeric[], integer, boolean) AS ASSIGNMENT;
|
||||
|
||||
-- minivec operators
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <#> (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_negative_inner_product,
|
||||
COMMUTATOR = '<#>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <=> (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = cosine_distance,
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <+> (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = l1_distance,
|
||||
COMMUTATOR = '<+>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR + (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_add,
|
||||
COMMUTATOR = +
|
||||
);
|
||||
|
||||
CREATE OPERATOR - (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_sub
|
||||
);
|
||||
|
||||
CREATE OPERATOR * (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_mul,
|
||||
COMMUTATOR = *
|
||||
);
|
||||
|
||||
CREATE OPERATOR || (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_concat
|
||||
);
|
||||
|
||||
CREATE OPERATOR < (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_lt,
|
||||
COMMUTATOR = > , NEGATOR = >= ,
|
||||
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
|
||||
);
|
||||
|
||||
CREATE OPERATOR <= (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_le,
|
||||
COMMUTATOR = >= , NEGATOR = > ,
|
||||
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
|
||||
);
|
||||
|
||||
CREATE OPERATOR = (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_eq,
|
||||
COMMUTATOR = = , NEGATOR = <> ,
|
||||
RESTRICT = eqsel, JOIN = eqjoinsel
|
||||
);
|
||||
|
||||
CREATE OPERATOR <> (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_ne,
|
||||
COMMUTATOR = <> , NEGATOR = = ,
|
||||
RESTRICT = eqsel, JOIN = eqjoinsel
|
||||
);
|
||||
|
||||
CREATE OPERATOR >= (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_ge,
|
||||
COMMUTATOR = <= , NEGATOR = < ,
|
||||
RESTRICT = scalargesel, JOIN = scalargejoinsel
|
||||
);
|
||||
|
||||
CREATE OPERATOR > (
|
||||
LEFTARG = minivec, RIGHTARG = minivec, PROCEDURE = minivec_gt,
|
||||
COMMUTATOR = < , NEGATOR = <= ,
|
||||
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
|
||||
);
|
||||
|
||||
-- minivec op classes
|
||||
|
||||
CREATE OPERATOR CLASS minivec_ops
|
||||
DEFAULT FOR TYPE minivec USING btree AS
|
||||
OPERATOR 1 < ,
|
||||
OPERATOR 2 <= ,
|
||||
OPERATOR 3 = ,
|
||||
OPERATOR 4 >= ,
|
||||
OPERATOR 5 > ,
|
||||
FUNCTION 1 minivec_cmp(minivec, minivec);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_l2_ops
|
||||
FOR TYPE minivec USING ivfflat AS
|
||||
OPERATOR 1 <-> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 minivec_l2_squared_distance(minivec, minivec),
|
||||
FUNCTION 3 l2_distance(minivec, minivec),
|
||||
FUNCTION 5 ivfflat_minivec_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_ip_ops
|
||||
FOR TYPE minivec USING ivfflat AS
|
||||
OPERATOR 1 <#> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 minivec_negative_inner_product(minivec, minivec),
|
||||
FUNCTION 3 minivec_spherical_distance(minivec, minivec),
|
||||
FUNCTION 4 l2_norm(minivec),
|
||||
FUNCTION 5 ivfflat_minivec_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_cosine_ops
|
||||
FOR TYPE minivec USING ivfflat AS
|
||||
OPERATOR 1 <=> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 cosine_distance(minivec, minivec),
|
||||
FUNCTION 2 l2_norm(minivec),
|
||||
FUNCTION 3 minivec_spherical_distance(minivec, minivec),
|
||||
FUNCTION 4 l2_norm(minivec),
|
||||
FUNCTION 5 ivfflat_minivec_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_l2_ops
|
||||
FOR TYPE minivec USING hnsw AS
|
||||
OPERATOR 1 <-> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 minivec_l2_squared_distance(minivec, minivec),
|
||||
FUNCTION 3 hnsw_minivec_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_ip_ops
|
||||
FOR TYPE minivec USING hnsw AS
|
||||
OPERATOR 1 <#> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 minivec_negative_inner_product(minivec, minivec),
|
||||
FUNCTION 3 hnsw_minivec_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_cosine_ops
|
||||
FOR TYPE minivec USING hnsw AS
|
||||
OPERATOR 1 <=> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 cosine_distance(minivec, minivec),
|
||||
FUNCTION 2 l2_norm(minivec),
|
||||
FUNCTION 3 hnsw_minivec_support(internal);
|
||||
|
||||
CREATE OPERATOR CLASS minivec_l1_ops
|
||||
FOR TYPE minivec USING hnsw AS
|
||||
OPERATOR 1 <+> (minivec, minivec) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 l1_distance(minivec, minivec),
|
||||
FUNCTION 3 hnsw_minivec_support(internal);
|
||||
|
||||
-- bit functions
|
||||
|
||||
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
|
||||
98
src/hnsw.c
98
src/hnsw.c
@@ -12,12 +12,15 @@
|
||||
#include "utils/float.h"
|
||||
#include "utils/guc.h"
|
||||
#include "utils/selfuncs.h"
|
||||
#include "utils/spccache.h"
|
||||
|
||||
#if PG_VERSION_NUM < 150000
|
||||
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
|
||||
#endif
|
||||
|
||||
int hnsw_ef_search;
|
||||
int hnsw_ef_stream;
|
||||
bool hnsw_streaming;
|
||||
int hnsw_lock_tranche_id;
|
||||
static relopt_kind hnsw_relopt_kind;
|
||||
|
||||
@@ -68,6 +71,17 @@ HnswInit(void)
|
||||
"Valid range is 1..1000.", &hnsw_ef_search,
|
||||
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
|
||||
/* TODO Figure out name */
|
||||
DefineCustomBoolVariable("hnsw.streaming", "Use streaming mode",
|
||||
NULL, &hnsw_streaming,
|
||||
HNSW_DEFAULT_STREAMING, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
|
||||
/* TODO Figure out name */
|
||||
/* TODO Use same value as ivfflat.max_probes for "all" */
|
||||
DefineCustomIntVariable("hnsw.ef_stream", "Sets the max number of additional candidates to visit for streaming search",
|
||||
"-1 means all", &hnsw_ef_stream,
|
||||
HNSW_DEFAULT_EF_STREAM, HNSW_MIN_EF_STREAM, HNSW_MAX_EF_STREAM, PGC_USERSET, 0, NULL, NULL, NULL);
|
||||
|
||||
MarkGUCPrefixReserved("hnsw");
|
||||
}
|
||||
|
||||
@@ -88,6 +102,33 @@ hnswbuildphasename(int64 phasenum)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Estimate ef needed for iterative scans
|
||||
*/
|
||||
static int
|
||||
EstimateEf(PlannerInfo *root, IndexPath *path)
|
||||
{
|
||||
double selectivity = 1;
|
||||
ListCell *lc;
|
||||
|
||||
/* Cannot estimate without limit */
|
||||
/* limit_tuples includes offset */
|
||||
if (root->limit_tuples < 0)
|
||||
return 0;
|
||||
|
||||
/* Get the selectivity of non-index conditions */
|
||||
foreach(lc, path->indexinfo->indrestrictinfo)
|
||||
{
|
||||
RestrictInfo *rinfo = lfirst(lc);
|
||||
|
||||
/* Skip DEFAULT_INEQ_SEL since it may be a distance filter */
|
||||
if (rinfo->norm_selec >= 0 && rinfo->norm_selec <= 1 && rinfo->norm_selec != (Selectivity) DEFAULT_INEQ_SEL)
|
||||
selectivity *= rinfo->norm_selec;
|
||||
}
|
||||
|
||||
return root->limit_tuples / Max(selectivity, 0.00001);
|
||||
}
|
||||
|
||||
/*
|
||||
* Estimate the cost of an index scan
|
||||
*/
|
||||
@@ -99,7 +140,12 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
{
|
||||
GenericCosts costs;
|
||||
int m;
|
||||
int ef;
|
||||
int entryLevel;
|
||||
int layer0TuplesMax;
|
||||
double layer0Selectivity;
|
||||
double scalingFactor = 0.55;
|
||||
double spc_seq_page_cost;
|
||||
Relation index;
|
||||
|
||||
/* Never use index without order */
|
||||
@@ -119,15 +165,57 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
HnswGetMetaPageInfo(index, &m, NULL);
|
||||
index_close(index, NoLock);
|
||||
|
||||
/* Approximate entry level */
|
||||
entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
|
||||
ef = hnsw_streaming ? Max(hnsw_ef_search, EstimateEf(root, path)) : hnsw_ef_search;
|
||||
|
||||
/* TODO Improve estimate of visited tuples (currently underestimates) */
|
||||
/* Account for number of tuples (or entry level), m, and ef_search */
|
||||
costs.numIndexTuples = (entryLevel + 2) * m;
|
||||
/*
|
||||
* HNSW cost estimation follows a formula that accounts for the total
|
||||
* number of tuples indexed combined with the parameters that most
|
||||
* influence the duration of the index scan, namely: m - the number of
|
||||
* tuples that are scanned in each step of the HNSW graph traversal
|
||||
* ef_search - which influences the total number of steps taken at layer 0
|
||||
*
|
||||
* The source of the vector data can impact how many steps it takes to
|
||||
* converge on the set of vectors to return to the executor. Currently, we
|
||||
* use a hardcoded scaling factor (HNSWScanScalingFactor) to help
|
||||
* influence that, but this could later become a configurable parameter
|
||||
* based on the cost estimations.
|
||||
*
|
||||
* The tuple estimator formula is below:
|
||||
*
|
||||
* numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
|
||||
*
|
||||
* "entryLevel * m" represents the floor of tuples we need to scan to get
|
||||
* to layer 0 (L0).
|
||||
*
|
||||
* "layer0TuplesMax" is the estimated total number of tuples we'd scan at
|
||||
* L0 if we weren't discarding already visited tuples as part of the scan.
|
||||
*
|
||||
* "layer0Selectivity" estimates the percentage of tuples that are scanned
|
||||
* at L0, accounting for previously visited tuples, multiplied by the
|
||||
* "scalingFactor" (currently hardcoded).
|
||||
*/
|
||||
entryLevel = (int) (log(path->indexinfo->tuples + 1) * HnswGetMl(m));
|
||||
layer0TuplesMax = HnswGetLayerM(m, 0) * ef;
|
||||
layer0Selectivity = (scalingFactor * log(path->indexinfo->tuples + 1)) /
|
||||
(log(m) * (1 + log(ef)));
|
||||
|
||||
costs.numIndexTuples = (entryLevel * m) +
|
||||
(layer0TuplesMax * layer0Selectivity);
|
||||
|
||||
genericcostestimate(root, path, loop_count, &costs);
|
||||
|
||||
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
|
||||
|
||||
/* Adjust cost if needed since TOAST not included in seq scan cost */
|
||||
if (costs.numIndexPages > path->indexinfo->rel->pages)
|
||||
{
|
||||
/* Change all page cost from random to sequential */
|
||||
costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
|
||||
/* Remove cost of extra pages */
|
||||
costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
|
||||
}
|
||||
|
||||
/* Use total cost since most work happens before first tuple is returned */
|
||||
*indexStartupCost = costs.indexTotalCost;
|
||||
*indexTotalCost = costs.indexTotalCost;
|
||||
|
||||
50
src/hnsw.h
50
src/hnsw.h
@@ -12,6 +12,10 @@
|
||||
#include "utils/sampling.h"
|
||||
#include "vector.h"
|
||||
|
||||
#ifdef HNSW_BENCH
|
||||
#include "portability/instr_time.h"
|
||||
#endif
|
||||
|
||||
#define HNSW_MAX_DIM 2000
|
||||
#define HNSW_MAX_NNZ 1000
|
||||
|
||||
@@ -42,6 +46,10 @@
|
||||
#define HNSW_DEFAULT_EF_SEARCH 40
|
||||
#define HNSW_MIN_EF_SEARCH 1
|
||||
#define HNSW_MAX_EF_SEARCH 1000
|
||||
#define HNSW_DEFAULT_STREAMING false
|
||||
#define HNSW_DEFAULT_EF_STREAM -1
|
||||
#define HNSW_MIN_EF_STREAM -1
|
||||
#define HNSW_MAX_EF_STREAM INT_MAX
|
||||
|
||||
/* Tuple types */
|
||||
#define HNSW_ELEMENT_TUPLE_TYPE 1
|
||||
@@ -68,6 +76,21 @@
|
||||
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
|
||||
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
|
||||
|
||||
#ifdef HNSW_BENCH
|
||||
#define HnswBench(name, code) \
|
||||
do { \
|
||||
instr_time start; \
|
||||
instr_time duration; \
|
||||
INSTR_TIME_SET_CURRENT(start); \
|
||||
(code); \
|
||||
INSTR_TIME_SET_CURRENT(duration); \
|
||||
INSTR_TIME_SUBTRACT(duration, start); \
|
||||
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
|
||||
} while (0)
|
||||
#else
|
||||
#define HnswBench(name, code) (code)
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 150000
|
||||
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
|
||||
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
|
||||
@@ -106,6 +129,8 @@
|
||||
|
||||
/* Variables */
|
||||
extern int hnsw_ef_search;
|
||||
extern int hnsw_ef_stream;
|
||||
extern bool hnsw_streaming;
|
||||
extern int hnsw_lock_tranche_id;
|
||||
|
||||
typedef struct HnswElementData HnswElementData;
|
||||
@@ -129,6 +154,7 @@ struct HnswElementData
|
||||
uint8 heaptidsLength;
|
||||
uint8 level;
|
||||
uint8 deleted;
|
||||
uint8 version;
|
||||
uint32 hash;
|
||||
HnswNeighborsPtr neighbors;
|
||||
BlockNumber blkno;
|
||||
@@ -163,6 +189,9 @@ typedef struct HnswSearchCandidate
|
||||
float distance;
|
||||
} HnswSearchCandidate;
|
||||
|
||||
#define HnswGetSearchCandidate(membername, ptr) pairingheap_container(HnswSearchCandidate, membername, ptr)
|
||||
#define HnswGetSearchCandidateConst(membername, ptr) pairingheap_const_container(HnswSearchCandidate, membername, ptr)
|
||||
|
||||
/* HNSW index options */
|
||||
typedef struct HnswOptions
|
||||
{
|
||||
@@ -306,10 +335,10 @@ typedef struct HnswElementTupleData
|
||||
uint8 type;
|
||||
uint8 level;
|
||||
uint8 deleted;
|
||||
uint8 unused;
|
||||
uint8 version;
|
||||
ItemPointerData heaptids[HNSW_HEAPTIDS];
|
||||
ItemPointerData neighbortid;
|
||||
uint16 unused2;
|
||||
uint16 unused;
|
||||
Vector data;
|
||||
} HnswElementTupleData;
|
||||
|
||||
@@ -318,18 +347,31 @@ typedef HnswElementTupleData * HnswElementTuple;
|
||||
typedef struct HnswNeighborTupleData
|
||||
{
|
||||
uint8 type;
|
||||
uint8 unused;
|
||||
uint8 version;
|
||||
uint16 count;
|
||||
ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
|
||||
} HnswNeighborTupleData;
|
||||
|
||||
typedef HnswNeighborTupleData * HnswNeighborTuple;
|
||||
|
||||
typedef union
|
||||
{
|
||||
struct pointerhash_hash *pointers;
|
||||
struct offsethash_hash *offsets;
|
||||
struct tidhash_hash *tids;
|
||||
} visited_hash;
|
||||
|
||||
typedef struct HnswScanOpaqueData
|
||||
{
|
||||
const HnswTypeInfo *typeInfo;
|
||||
bool first;
|
||||
List *w;
|
||||
visited_hash v;
|
||||
pairingheap *discarded;
|
||||
Datum q;
|
||||
int m;
|
||||
int64 tuples;
|
||||
double previousDistance;
|
||||
MemoryContext tmpCtx;
|
||||
|
||||
/* Support functions */
|
||||
@@ -375,7 +417,7 @@ bool HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
|
||||
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
||||
void HnswInitPage(Buffer buf, Page page);
|
||||
void HnswInit(void);
|
||||
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
|
||||
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
|
||||
HnswElement HnswGetEntryPoint(Relation index);
|
||||
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
|
||||
void *HnswAlloc(HnswAllocator * allocator, Size size);
|
||||
|
||||
@@ -36,7 +36,7 @@ GetInsertPage(Relation index)
|
||||
* Check for a free offset
|
||||
*/
|
||||
static bool
|
||||
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
|
||||
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage, uint8 *tupleVersion)
|
||||
{
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
|
||||
@@ -98,6 +98,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
|
||||
{
|
||||
*freeOffno = offno;
|
||||
*freeNeighborOffno = neighborOffno;
|
||||
*tupleVersion = etup->version;
|
||||
return true;
|
||||
}
|
||||
else if (*nbuf != buf)
|
||||
@@ -153,6 +154,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
|
||||
OffsetNumber freeOffno = InvalidOffsetNumber;
|
||||
OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
|
||||
BlockNumber newInsertPage = InvalidBlockNumber;
|
||||
uint8 tupleVersion;
|
||||
char *base = NULL;
|
||||
|
||||
/* Calculate sizes */
|
||||
@@ -202,7 +204,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
|
||||
}
|
||||
|
||||
/* Next, try space from a deleted element */
|
||||
if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
|
||||
if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage, &tupleVersion))
|
||||
{
|
||||
if (nbuf != buf)
|
||||
{
|
||||
@@ -212,6 +214,10 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
|
||||
npage = GenericXLogRegisterBuffer(state, nbuf, 0);
|
||||
}
|
||||
|
||||
/* Set tuple version */
|
||||
etup->version = tupleVersion;
|
||||
ntup->version = tupleVersion;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
135
src/hnswscan.c
135
src/hnswscan.c
@@ -1,5 +1,7 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include <float.h>
|
||||
|
||||
#include "access/relscan.h"
|
||||
#include "hnsw.h"
|
||||
#include "pgstat.h"
|
||||
@@ -26,6 +28,9 @@ GetScanItems(IndexScanDesc scan, Datum q)
|
||||
/* Get m and entry point */
|
||||
HnswGetMetaPageInfo(index, &m, &entryPoint);
|
||||
|
||||
so->q = q;
|
||||
so->m = m;
|
||||
|
||||
if (entryPoint == NULL)
|
||||
return NIL;
|
||||
|
||||
@@ -33,11 +38,44 @@ GetScanItems(IndexScanDesc scan, Datum q)
|
||||
|
||||
for (int lc = entryPoint->level; lc >= 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL, NULL, NULL, true, NULL);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
|
||||
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, &so->v, hnsw_streaming ? &so->discarded : NULL, true, &so->tuples);
|
||||
}
|
||||
|
||||
/*
|
||||
* Resume scan at ground level with discarded candidates
|
||||
*/
|
||||
static List *
|
||||
ResumeScanItems(IndexScanDesc scan)
|
||||
{
|
||||
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
|
||||
Relation index = scan->indexRelation;
|
||||
FmgrInfo *procinfo = so->procinfo;
|
||||
Oid collation = so->collation;
|
||||
List *ep = NIL;
|
||||
char *base = NULL;
|
||||
int batch_size = hnsw_ef_search;
|
||||
|
||||
if (pairingheap_is_empty(so->discarded))
|
||||
return NIL;
|
||||
|
||||
/* Get next batch of candidates */
|
||||
for (int i = 0; i < batch_size; i++)
|
||||
{
|
||||
HnswSearchCandidate *hc;
|
||||
|
||||
if (pairingheap_is_empty(so->discarded))
|
||||
break;
|
||||
|
||||
hc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded));
|
||||
|
||||
ep = lappend(ep, hc);
|
||||
}
|
||||
|
||||
return HnswSearchLayer(base, so->q, ep, batch_size, 0, index, procinfo, collation, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -81,6 +119,8 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
|
||||
so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
|
||||
so->typeInfo = HnswGetTypeInfo(index);
|
||||
so->first = true;
|
||||
so->v.tids = NULL;
|
||||
so->discarded = NULL;
|
||||
so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
|
||||
"Hnsw scan temporary context",
|
||||
ALLOCSET_DEFAULT_SIZES);
|
||||
@@ -103,7 +143,15 @@ hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int no
|
||||
{
|
||||
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
|
||||
|
||||
if (so->v.tids != NULL)
|
||||
tidhash_reset(so->v.tids);
|
||||
|
||||
if (so->discarded != NULL)
|
||||
pairingheap_reset(so->discarded);
|
||||
|
||||
so->first = true;
|
||||
so->tuples = 0;
|
||||
so->previousDistance = -INFINITY;
|
||||
MemoryContextReset(so->tmpCtx);
|
||||
|
||||
if (keys && scan->numberOfKeys > 0)
|
||||
@@ -153,7 +201,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
*/
|
||||
LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
|
||||
|
||||
so->w = GetScanItems(scan, value);
|
||||
HnswBench("scan iteration", so->w = GetScanItems(scan, value));
|
||||
|
||||
/* Release shared lock */
|
||||
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
|
||||
@@ -165,22 +213,97 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
#endif
|
||||
}
|
||||
|
||||
while (list_length(so->w) > 0)
|
||||
for (;;)
|
||||
{
|
||||
char *base = NULL;
|
||||
HnswSearchCandidate *hc = llast(so->w);
|
||||
HnswElement element = HnswPtrAccess(base, hc->element);
|
||||
HnswSearchCandidate *hc;
|
||||
HnswElement element;
|
||||
ItemPointer heaptid;
|
||||
|
||||
if (list_length(so->w) == 0)
|
||||
{
|
||||
if (!hnsw_streaming)
|
||||
break;
|
||||
|
||||
/* Empty index */
|
||||
if (so->discarded == NULL)
|
||||
break;
|
||||
|
||||
/* Reached max number of additional tuples */
|
||||
if (hnsw_ef_stream != -1 && so->tuples >= hnsw_ef_search + hnsw_ef_stream)
|
||||
{
|
||||
if (pairingheap_is_empty(so->discarded))
|
||||
break;
|
||||
|
||||
/* Return remaining tuples */
|
||||
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
|
||||
}
|
||||
/* Prevent scans from consuming too much memory */
|
||||
else if (MemoryContextMemAllocated(so->tmpCtx, false) > (Size) work_mem * 1024L)
|
||||
{
|
||||
if (pairingheap_is_empty(so->discarded))
|
||||
{
|
||||
ereport(NOTICE,
|
||||
(errmsg("hnsw index scan exceeded work_mem after " INT64_FORMAT " tuples", so->tuples),
|
||||
errhint("Increase work_mem to scan more tuples.")));
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
/* Return remaining tuples */
|
||||
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
|
||||
}
|
||||
else
|
||||
{
|
||||
/*
|
||||
* Locking ensures when neighbors are read, the elements they
|
||||
* reference will not be deleted (and replaced) during the
|
||||
* iteration.
|
||||
*
|
||||
* Elements loaded into memory on previous iterations may have
|
||||
* been deleted (and replaced), so when reading neighbors, the
|
||||
* element version must be checked.
|
||||
*/
|
||||
LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
|
||||
|
||||
HnswBench("scan iteration", so->w = ResumeScanItems(scan));
|
||||
|
||||
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
|
||||
|
||||
#if defined(HNSW_MEMORY)
|
||||
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024);
|
||||
#endif
|
||||
}
|
||||
|
||||
if (list_length(so->w) == 0)
|
||||
break;
|
||||
}
|
||||
|
||||
hc = llast(so->w);
|
||||
element = HnswPtrAccess(base, hc->element);
|
||||
|
||||
/* Move to next element if no valid heap TIDs */
|
||||
if (element->heaptidsLength == 0)
|
||||
{
|
||||
so->w = list_delete_last(so->w);
|
||||
|
||||
/* Mark memory as free for next iteration */
|
||||
if (hnsw_streaming)
|
||||
{
|
||||
pfree(element);
|
||||
pfree(hc);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
heaptid = &element->heaptids[--element->heaptidsLength];
|
||||
|
||||
if (hc->distance < so->previousDistance)
|
||||
continue;
|
||||
|
||||
so->previousDistance = hc->distance;
|
||||
|
||||
MemoryContextSwitchTo(oldCtx);
|
||||
|
||||
scan->xs_heaptid = *heaptid;
|
||||
|
||||
128
src/hnswutils.c
128
src/hnswutils.c
@@ -100,13 +100,6 @@ hash_offset(Size offset)
|
||||
#define SH_DEFINE
|
||||
#include "lib/simplehash.h"
|
||||
|
||||
typedef union
|
||||
{
|
||||
pointerhash_hash *pointers;
|
||||
offsethash_hash *offsets;
|
||||
tidhash_hash *tids;
|
||||
} visited_hash;
|
||||
|
||||
typedef union
|
||||
{
|
||||
HnswElement element;
|
||||
@@ -159,9 +152,6 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
|
||||
Datum
|
||||
HnswNormValue(const HnswTypeInfo * typeInfo, Oid collation, Datum value)
|
||||
{
|
||||
if (!typeInfo->normalize)
|
||||
return value;
|
||||
|
||||
return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
|
||||
}
|
||||
|
||||
@@ -256,6 +246,8 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
|
||||
|
||||
element->level = level;
|
||||
element->deleted = 0;
|
||||
/* Start at one to make it easier to find issues */
|
||||
element->version = 1;
|
||||
|
||||
HnswInitNeighbors(base, element, m, allocator);
|
||||
|
||||
@@ -408,6 +400,7 @@ HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element)
|
||||
etup->type = HNSW_ELEMENT_TUPLE_TYPE;
|
||||
etup->level = element->level;
|
||||
etup->deleted = 0;
|
||||
etup->version = element->version;
|
||||
for (int i = 0; i < HNSW_HEAPTIDS; i++)
|
||||
{
|
||||
if (i < element->heaptidsLength)
|
||||
@@ -450,6 +443,7 @@ HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m)
|
||||
}
|
||||
|
||||
ntup->count = idx;
|
||||
ntup->version = e->version;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -523,6 +517,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
|
||||
{
|
||||
element->level = etup->level;
|
||||
element->deleted = etup->deleted;
|
||||
element->version = etup->version;
|
||||
element->neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
|
||||
element->neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
|
||||
element->heaptidsLength = 0;
|
||||
@@ -624,9 +619,6 @@ HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index,
|
||||
return hc;
|
||||
}
|
||||
|
||||
#define HnswGetSearchCandidate(membername, ptr) pairingheap_container(HnswSearchCandidate, membername, ptr)
|
||||
#define HnswGetSearchCandidateConst(membername, ptr) pairingheap_const_container(HnswSearchCandidate, membername, ptr)
|
||||
|
||||
/*
|
||||
* Compare candidate distances
|
||||
*/
|
||||
@@ -642,6 +634,21 @@ CompareNearestCandidates(const pairingheap_node *a, const pairingheap_node *b, v
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compare discarded candidate distances
|
||||
*/
|
||||
static int
|
||||
CompareNearestDiscardedCandidates(const pairingheap_node *a, const pairingheap_node *b, void *arg)
|
||||
{
|
||||
if (HnswGetSearchCandidateConst(w_node, a)->distance < HnswGetSearchCandidateConst(w_node, b)->distance)
|
||||
return 1;
|
||||
|
||||
if (HnswGetSearchCandidateConst(w_node, a)->distance > HnswGetSearchCandidateConst(w_node, b)->distance)
|
||||
return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compare candidate distances
|
||||
*/
|
||||
@@ -757,20 +764,30 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
|
||||
int start;
|
||||
ItemPointerData indextids[HNSW_MAX_M * 2];
|
||||
|
||||
*unvisitedLength = 0;
|
||||
|
||||
buf = ReadBuffer(index, element->neighborPage);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
|
||||
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
|
||||
start = (element->level - lc) * m;
|
||||
|
||||
/*
|
||||
* Ensure the neighbor tuple has not been deleted or replaced between
|
||||
* index scan iterations
|
||||
*/
|
||||
if (ntup->version != element->version || ntup->count != (element->level + 2) * m)
|
||||
{
|
||||
UnlockReleaseBuffer(buf);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Copy to minimize lock time */
|
||||
start = (element->level - lc) * m;
|
||||
memcpy(&indextids, ntup->indextids + start, lm * sizeof(ItemPointerData));
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
*unvisitedLength = 0;
|
||||
|
||||
for (int i = 0; i < lm; i++)
|
||||
{
|
||||
ItemPointer indextid = &indextids[i];
|
||||
@@ -790,13 +807,13 @@ HnswLoadUnvisitedFromDisk(HnswElement element, HnswUnvisited * unvisited, int *u
|
||||
* Algorithm 2 from paper
|
||||
*/
|
||||
List *
|
||||
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
|
||||
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples)
|
||||
{
|
||||
List *w = NIL;
|
||||
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
|
||||
pairingheap *W = pairingheap_allocate(CompareFurthestCandidates, NULL);
|
||||
int wlen = 0;
|
||||
visited_hash v;
|
||||
visited_hash vh;
|
||||
ListCell *lc2;
|
||||
HnswNeighborArray *localNeighborhood = NULL;
|
||||
Size neighborhoodSize = 0;
|
||||
@@ -804,7 +821,19 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
|
||||
HnswUnvisited *unvisited = palloc(lm * sizeof(HnswUnvisited));
|
||||
int unvisitedLength;
|
||||
|
||||
InitVisited(base, &v, index, ef, m);
|
||||
if (v == NULL)
|
||||
{
|
||||
v = &vh;
|
||||
initVisited = true;
|
||||
}
|
||||
|
||||
if (initVisited)
|
||||
{
|
||||
InitVisited(base, v, index, ef, m);
|
||||
|
||||
if (discarded != NULL)
|
||||
*discarded = pairingheap_allocate(CompareNearestDiscardedCandidates, NULL);
|
||||
}
|
||||
|
||||
/* Create local memory for neighborhood if needed */
|
||||
if (index == NULL)
|
||||
@@ -819,7 +848,13 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
|
||||
HnswSearchCandidate *hc = (HnswSearchCandidate *) lfirst(lc2);
|
||||
bool found;
|
||||
|
||||
AddToVisited(base, &v, hc->element, index, &found);
|
||||
if (initVisited)
|
||||
{
|
||||
AddToVisited(base, v, hc->element, index, &found);
|
||||
|
||||
if (tuples != NULL)
|
||||
(*tuples)++;
|
||||
}
|
||||
|
||||
pairingheap_add(C, &hc->c_node);
|
||||
pairingheap_add(W, &hc->w_node);
|
||||
@@ -845,9 +880,12 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
|
||||
cElement = HnswPtrAccess(base, c->element);
|
||||
|
||||
if (index == NULL)
|
||||
HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, &v, lc, localNeighborhood, neighborhoodSize);
|
||||
HnswLoadUnvisitedFromMemory(base, cElement, unvisited, &unvisitedLength, v, lc, localNeighborhood, neighborhoodSize);
|
||||
else
|
||||
HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, &v, index, m, lm, lc);
|
||||
HnswLoadUnvisitedFromDisk(cElement, unvisited, &unvisitedLength, v, index, m, lm, lc);
|
||||
|
||||
if (tuples != NULL)
|
||||
(*tuples) += unvisitedLength;
|
||||
|
||||
for (int i = 0; i < unvisitedLength; i++)
|
||||
{
|
||||
@@ -871,16 +909,22 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
|
||||
|
||||
/* Avoid any allocations if not adding */
|
||||
eElement = NULL;
|
||||
HnswLoadElementImpl(blkno, offno, &eDistance, &q, index, procinfo, collation, inserting, alwaysAdd ? NULL : &f->distance, &eElement);
|
||||
|
||||
if (eElement == NULL)
|
||||
continue;
|
||||
HnswLoadElementImpl(blkno, offno, &eDistance, &q, index, procinfo, collation, inserting, alwaysAdd || discarded != NULL ? NULL : &f->distance, &eElement);
|
||||
}
|
||||
|
||||
if (!(eDistance < f->distance || alwaysAdd))
|
||||
continue;
|
||||
if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
|
||||
{
|
||||
if (discarded != NULL)
|
||||
{
|
||||
/* Create a new candidate */
|
||||
e = palloc(sizeof(HnswSearchCandidate));
|
||||
HnswPtrStore(base, e->element, eElement);
|
||||
e->distance = eDistance;
|
||||
pairingheap_add(*discarded, &e->w_node);
|
||||
}
|
||||
|
||||
Assert(!eElement->deleted);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Make robust to issues */
|
||||
if (eElement->level < lc)
|
||||
@@ -904,7 +948,12 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
|
||||
|
||||
/* No need to decrement wlen */
|
||||
if (wlen > ef)
|
||||
pairingheap_remove_first(W);
|
||||
{
|
||||
HnswSearchCandidate *d = HnswGetSearchCandidate(w_node, pairingheap_remove_first(W));
|
||||
|
||||
if (discarded != NULL)
|
||||
pairingheap_add(*discarded, &d->w_node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1277,7 +1326,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
/* 1st phase: greedy search to insert level */
|
||||
for (int lc = entryLevel; lc >= level + 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement, NULL, NULL, true, NULL);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
@@ -1296,7 +1345,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
List *lw = NIL;
|
||||
ListCell *lc2;
|
||||
|
||||
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
|
||||
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement, NULL, NULL, true, NULL);
|
||||
|
||||
/* Convert search candidates to candidates */
|
||||
foreach(lc2, w)
|
||||
@@ -1330,7 +1379,6 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
|
||||
PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
|
||||
PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
PGDLLEXPORT Datum minivec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
|
||||
static void
|
||||
@@ -1379,20 +1427,6 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_minivec_support);
|
||||
Datum
|
||||
hnsw_minivec_support(PG_FUNCTION_ARGS)
|
||||
{
|
||||
static const HnswTypeInfo typeInfo = {
|
||||
.maxDimensions = HNSW_MAX_DIM * 4,
|
||||
/* Do not normalize to maximize precision */
|
||||
.normalize = NULL,
|
||||
.checkValue = NULL
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnsw_bit_support);
|
||||
Datum
|
||||
hnsw_bit_support(PG_FUNCTION_ARGS)
|
||||
|
||||
@@ -527,6 +527,14 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
for (int i = 0; i < ntup->count; i++)
|
||||
ItemPointerSetInvalid(&ntup->indextids[i]);
|
||||
|
||||
/* Increment version */
|
||||
/* This is used to avoid incorrect reads for iterative scans */
|
||||
/* Reserve some bits for future use */
|
||||
etup->version++;
|
||||
if (etup->version > 15)
|
||||
etup->version = 1;
|
||||
ntup->version = etup->version;
|
||||
|
||||
/*
|
||||
* We modified the tuples in place, no need to call
|
||||
* PageIndexTupleOverwrite
|
||||
|
||||
@@ -120,13 +120,6 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the list selectivity is lower than what is returned from the generic
|
||||
* cost estimator, use that.
|
||||
*/
|
||||
if (ratio < costs.indexSelectivity)
|
||||
costs.indexSelectivity = ratio;
|
||||
|
||||
/* Use total cost since most work happens before first tuple is returned */
|
||||
*indexStartupCost = costs.indexTotalCost;
|
||||
*indexTotalCost = costs.indexTotalCost;
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#include "halfutils.h"
|
||||
#include "halfvec.h"
|
||||
#include "ivfflat.h"
|
||||
#include "minivec.h"
|
||||
#include "storage/bufmgr.h"
|
||||
|
||||
/*
|
||||
@@ -71,9 +70,6 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
|
||||
Datum
|
||||
IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value)
|
||||
{
|
||||
if (!typeInfo->normalize)
|
||||
return value;
|
||||
|
||||
return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
|
||||
}
|
||||
|
||||
@@ -235,7 +231,6 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
|
||||
|
||||
PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
|
||||
PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
PGDLLEXPORT Datum minivec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
|
||||
|
||||
static Size
|
||||
@@ -250,12 +245,6 @@ HalfvecItemSize(int dimensions)
|
||||
return HALFVEC_SIZE(dimensions);
|
||||
}
|
||||
|
||||
static Size
|
||||
MinivecItemSize(int dimensions)
|
||||
{
|
||||
return MINIVEC_SIZE(dimensions);
|
||||
}
|
||||
|
||||
static Size
|
||||
BitItemSize(int dimensions)
|
||||
{
|
||||
@@ -286,18 +275,6 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
|
||||
vec->x[k] = Float4ToHalfUnchecked(x[k]);
|
||||
}
|
||||
|
||||
static void
|
||||
MinivecUpdateCenter(Pointer v, int dimensions, float *x)
|
||||
{
|
||||
MiniVector *vec = (MiniVector *) v;
|
||||
|
||||
SET_VARSIZE(vec, MINIVEC_SIZE(dimensions));
|
||||
vec->dim = dimensions;
|
||||
|
||||
for (int k = 0; k < dimensions; k++)
|
||||
vec->x[k] = Float4ToFp8Unchecked(x[k]);
|
||||
}
|
||||
|
||||
static void
|
||||
BitUpdateCenter(Pointer v, int dimensions, float *x)
|
||||
{
|
||||
@@ -332,15 +309,6 @@ HalfvecSumCenter(Pointer v, float *x)
|
||||
x[k] += HalfToFloat4(vec->x[k]);
|
||||
}
|
||||
|
||||
static void
|
||||
MinivecSumCenter(Pointer v, float *x)
|
||||
{
|
||||
MiniVector *vec = (MiniVector *) v;
|
||||
|
||||
for (int k = 0; k < vec->dim; k++)
|
||||
x[k] += Fp8ToFloat4(vec->x[k]);
|
||||
}
|
||||
|
||||
static void
|
||||
BitSumCenter(Pointer v, float *x)
|
||||
{
|
||||
@@ -389,22 +357,6 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_minivec_support);
|
||||
Datum
|
||||
ivfflat_minivec_support(PG_FUNCTION_ARGS)
|
||||
{
|
||||
static const IvfflatTypeInfo typeInfo = {
|
||||
.maxDimensions = IVFFLAT_MAX_DIM * 4,
|
||||
/* Do not normalize to maximize precision */
|
||||
.normalize = NULL,
|
||||
.itemSize = MinivecItemSize,
|
||||
.updateCenter = MinivecUpdateCenter,
|
||||
.sumCenter = MinivecSumCenter
|
||||
};
|
||||
|
||||
PG_RETURN_POINTER(&typeInfo);
|
||||
};
|
||||
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
|
||||
Datum
|
||||
ivfflat_bit_support(PG_FUNCTION_ARGS)
|
||||
|
||||
1062
src/minivec.c
1062
src/minivec.c
File diff suppressed because it is too large
Load Diff
177
src/minivec.h
177
src/minivec.h
@@ -1,177 +0,0 @@
|
||||
#ifndef MINIVEC_H
|
||||
#define MINIVEC_H
|
||||
|
||||
#define MINIVEC_MAX_DIM 16000
|
||||
|
||||
#define fp8 uint8
|
||||
|
||||
#define MINIVEC_SIZE(_dim) (offsetof(MiniVector, x) + sizeof(fp8)*(_dim))
|
||||
#define DatumGetMiniVector(x) ((MiniVector *) PG_DETOAST_DATUM(x))
|
||||
#define PG_GETARG_MINIVEC_P(x) DatumGetMiniVector(PG_GETARG_DATUM(x))
|
||||
#define PG_RETURN_MINIVEC_P(x) PG_RETURN_POINTER(x)
|
||||
|
||||
typedef struct MiniVector
|
||||
{
|
||||
int32 vl_len_; /* varlena header (do not touch directly!) */
|
||||
int16 dim; /* number of dimensions */
|
||||
int16 unused; /* reserved for future use, always zero */
|
||||
fp8 x[FLEXIBLE_ARRAY_MEMBER];
|
||||
} MiniVector;
|
||||
|
||||
MiniVector *InitMiniVector(int dim);
|
||||
|
||||
/*
|
||||
* Check if fp8 is NaN
|
||||
*/
|
||||
static inline bool
|
||||
Fp8IsNan(fp8 num)
|
||||
{
|
||||
return (num & 0x7C) == 0x7C && (num & 0x7F) != 0x7C;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if fp8 is infinite
|
||||
*/
|
||||
static inline bool
|
||||
Fp8IsInf(fp8 num)
|
||||
{
|
||||
return (num & 0x7F) == 0x7C;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if fp8 is zero
|
||||
*/
|
||||
static inline bool
|
||||
Fp8IsZero(fp8 num)
|
||||
{
|
||||
return num == 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert a fp8 to a float4
|
||||
*/
|
||||
static inline float
|
||||
Fp8ToFloat4(fp8 num)
|
||||
{
|
||||
/* Lookup table for non-sign bits */
|
||||
/* Uses uint32 for correctness */
|
||||
uint32 lookup[128] = {0, 931135488, 939524096, 943718400, 947912704, 950009856, 952107008, 954204160, 956301312, 958398464, 960495616, 962592768, 964689920, 966787072, 968884224, 970981376, 973078528, 975175680, 977272832, 979369984, 981467136, 983564288, 985661440, 987758592, 989855744, 991952896, 994050048, 996147200, 998244352, 1000341504, 1002438656, 1004535808, 1006632960, 1008730112, 1010827264, 1012924416, 1015021568, 1017118720, 1019215872, 1021313024, 1023410176, 1025507328, 1027604480, 1029701632, 1031798784, 1033895936, 1035993088, 1038090240, 1040187392, 1042284544, 1044381696, 1046478848, 1048576000, 1050673152, 1052770304, 1054867456, 1056964608, 1059061760, 1061158912, 1063256064, 1065353216, 1067450368, 1069547520, 1071644672, 1073741824, 1075838976, 1077936128, 1080033280, 1082130432, 1084227584, 1086324736, 1088421888, 1090519040, 1092616192, 1094713344, 1096810496, 1098907648, 1101004800, 1103101952, 1105199104, 1107296256, 1109393408, 1111490560, 1113587712, 1115684864, 1117782016, 1119879168, 1121976320, 1124073472, 1126170624, 1128267776, 1130364928, 1132462080, 1134559232, 1136656384, 1138753536, 1140850688, 1142947840, 1145044992, 1147142144, 1149239296, 1151336448, 1153433600, 1155530752, 1157627904, 1159725056, 1161822208, 1163919360, 1166016512, 1168113664, 1170210816, 1172307968, 1174405120, 1176502272, 1178599424, 1180696576, 1182793728, 1184890880, 1186988032, 1189085184, 1191182336, 1193279488, 1195376640, 1197473792, 2139095040, 2145386496, 2143289344, 2145386496};
|
||||
|
||||
union
|
||||
{
|
||||
float f;
|
||||
uint32 i;
|
||||
} swap;
|
||||
|
||||
swap.i = lookup[num & 0x7F];
|
||||
|
||||
return (num & 0x80) == 0x80 ? -swap.f : swap.f;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert a float4 to a fp8
|
||||
*/
|
||||
static inline fp8
|
||||
Float4ToFp8Unchecked(float num)
|
||||
{
|
||||
union
|
||||
{
|
||||
float f;
|
||||
uint32 i;
|
||||
} swap;
|
||||
|
||||
uint32 bin;
|
||||
int exponent;
|
||||
int mantissa;
|
||||
uint8 result;
|
||||
|
||||
swap.f = num;
|
||||
bin = swap.i;
|
||||
exponent = (bin & 0x7F800000) >> 23;
|
||||
mantissa = bin & 0x007FFFFF;
|
||||
|
||||
/* Sign */
|
||||
result = (bin & 0x80000000) >> 24;
|
||||
|
||||
if (isinf(num))
|
||||
{
|
||||
/* Infinite */
|
||||
result |= 0x7C;
|
||||
}
|
||||
else if (isnan(num))
|
||||
{
|
||||
/* NaN */
|
||||
result |= 0x7F;
|
||||
}
|
||||
else if (exponent > 98)
|
||||
{
|
||||
int m;
|
||||
int gr;
|
||||
int s;
|
||||
|
||||
exponent -= 127;
|
||||
s = mantissa & 0x000FFFFF;
|
||||
|
||||
/* Subnormal */
|
||||
if (exponent < -14)
|
||||
{
|
||||
int diff = -exponent - 14;
|
||||
|
||||
mantissa >>= diff;
|
||||
mantissa += 1 << (23 - diff);
|
||||
s |= mantissa & 0x000FFFFF;
|
||||
}
|
||||
|
||||
m = mantissa >> 21;
|
||||
|
||||
/* Round */
|
||||
gr = (mantissa >> 20) % 4;
|
||||
if (gr == 3 || (gr == 1 && s != 0))
|
||||
m += 1;
|
||||
|
||||
if (m == 4)
|
||||
{
|
||||
m = 0;
|
||||
exponent += 1;
|
||||
}
|
||||
|
||||
if (exponent > 15)
|
||||
{
|
||||
/* Infinite */
|
||||
result |= 0x7C;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exponent >= -14)
|
||||
result |= (exponent + 15) << 2;
|
||||
|
||||
result |= m;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert a float4 to a fp8
|
||||
*/
|
||||
static inline fp8
|
||||
Float4ToFp8(float num)
|
||||
{
|
||||
fp8 result = Float4ToFp8Unchecked(num);
|
||||
|
||||
if (unlikely(Fp8IsInf(result)) && !isinf(num))
|
||||
{
|
||||
char *buf = palloc(FLOAT_SHORTEST_DECIMAL_LEN);
|
||||
|
||||
float_to_shortest_decimal_buf(num, buf);
|
||||
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("\"%s\" is out of range for type minivec", buf)));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif
|
||||
23
src/vector.c
23
src/vector.c
@@ -13,7 +13,6 @@
|
||||
#include "ivfflat.h"
|
||||
#include "lib/stringinfo.h"
|
||||
#include "libpq/pqformat.h"
|
||||
#include "minivec.h"
|
||||
#include "port.h" /* for strtof() */
|
||||
#include "sparsevec.h"
|
||||
#include "utils/array.h"
|
||||
@@ -543,28 +542,6 @@ halfvec_to_vector(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert fp8 vector to vector
|
||||
*/
|
||||
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(minivec_to_vector);
|
||||
Datum
|
||||
minivec_to_vector(PG_FUNCTION_ARGS)
|
||||
{
|
||||
MiniVector *vec = PG_GETARG_MINIVEC_P(0);
|
||||
int32 typmod = PG_GETARG_INT32(1);
|
||||
Vector *result;
|
||||
|
||||
CheckDim(vec->dim);
|
||||
CheckExpectedDim(typmod, vec->dim);
|
||||
|
||||
result = InitVector(vec->dim);
|
||||
|
||||
for (int i = 0; i < vec->dim; i++)
|
||||
result->x[i] = Fp8ToFloat4(vec->x[i]);
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
VECTOR_TARGET_CLONES static float
|
||||
VectorL2SquaredDistance(int dim, float *ax, float *bx)
|
||||
{
|
||||
|
||||
@@ -38,26 +38,6 @@ SELECT * FROM t ORDER BY val;
|
||||
|
||||
(4 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
-- minivec
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t (val);
|
||||
SELECT * FROM t WHERE val = '[1,2,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM t ORDER BY val;
|
||||
val
|
||||
---------
|
||||
[0,0,0]
|
||||
[1,1,1]
|
||||
[1,2,3]
|
||||
|
||||
(4 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
-- sparsevec
|
||||
CREATE TABLE t (val sparsevec(3));
|
||||
|
||||
@@ -140,64 +140,6 @@ SELECT '{1e-8,-1e-8}'::real[]::halfvec;
|
||||
[0,-0]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector::minivec;
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector::minivec(3);
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::vector::minivec(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '[61440]'::vector::minivec;
|
||||
ERROR: "61440" is out of range for type minivec
|
||||
SELECT '[1e-8]'::vector::minivec;
|
||||
minivec
|
||||
---------
|
||||
[0]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec::vector;
|
||||
vector
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec::vector(3);
|
||||
vector
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec::vector(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '{1,2,3}'::real[]::minivec;
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '{1,2,3}'::real[]::minivec(3);
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '{1,2,3}'::real[]::minivec(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '{61440,-61440}'::real[]::minivec;
|
||||
ERROR: "61440" is out of range for type minivec
|
||||
SELECT '{1e-8,-1e-8}'::real[]::minivec;
|
||||
minivec
|
||||
---------
|
||||
[0,-0]
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
|
||||
sparsevec
|
||||
-----------------
|
||||
|
||||
@@ -30,23 +30,6 @@ SELECT * FROM t2 ORDER BY val;
|
||||
|
||||
(4 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
DROP TABLE t2;
|
||||
-- minivec
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE TABLE t2 (val minivec(3));
|
||||
\copy t TO 'results/minivec.bin' WITH (FORMAT binary)
|
||||
\copy t2 FROM 'results/minivec.bin' WITH (FORMAT binary)
|
||||
SELECT * FROM t2 ORDER BY val;
|
||||
val
|
||||
---------
|
||||
[0,0,0]
|
||||
[1,1,1]
|
||||
[1,2,3]
|
||||
|
||||
(4 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
DROP TABLE t2;
|
||||
-- sparsevec
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
-- L2
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val minivec_l2_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
[1,1,1]
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::minivec)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
SELECT COUNT(*) FROM t;
|
||||
count
|
||||
-------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
TRUNCATE t;
|
||||
NOTICE: ivfflat index created with little data
|
||||
DETAIL: This will cause low recall.
|
||||
HINT: Drop the index until the table has more data.
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
-- inner product
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val minivec_ip_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,2,4]
|
||||
[1,2,3]
|
||||
[1,1,1]
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::minivec)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
-- cosine
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val minivec_cosine_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
|
||||
val
|
||||
---------
|
||||
[1,1,1]
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
(3 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
|
||||
count
|
||||
-------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::minivec)) t2;
|
||||
count
|
||||
-------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,590 +0,0 @@
|
||||
SELECT '[1,2,3]'::minivec;
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[-1,-2,-3]'::minivec;
|
||||
minivec
|
||||
------------
|
||||
[-1,-2,-3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.,2.,3.]'::minivec;
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT ' [ 1, 2 , 3 ] '::minivec;
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.23456]'::minivec;
|
||||
minivec
|
||||
---------
|
||||
[1.25]
|
||||
(1 row)
|
||||
|
||||
SELECT '[hello,1]'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[hello,1]"
|
||||
LINE 1: SELECT '[hello,1]'::minivec;
|
||||
^
|
||||
SELECT '[NaN,1]'::minivec;
|
||||
ERROR: NaN not allowed in minivec
|
||||
LINE 1: SELECT '[NaN,1]'::minivec;
|
||||
^
|
||||
SELECT '[Infinity,1]'::minivec;
|
||||
ERROR: infinite value not allowed in minivec
|
||||
LINE 1: SELECT '[Infinity,1]'::minivec;
|
||||
^
|
||||
SELECT '[-Infinity,1]'::minivec;
|
||||
ERROR: infinite value not allowed in minivec
|
||||
LINE 1: SELECT '[-Infinity,1]'::minivec;
|
||||
^
|
||||
SELECT '[61439,-61439]'::minivec;
|
||||
minivec
|
||||
----------------
|
||||
[57344,-57344]
|
||||
(1 row)
|
||||
|
||||
SELECT '[61440,-61440]'::minivec;
|
||||
ERROR: "61440" is out of range for type minivec
|
||||
LINE 1: SELECT '[61440,-61440]'::minivec;
|
||||
^
|
||||
SELECT '[1e-8,-1e-8]'::minivec;
|
||||
minivec
|
||||
---------
|
||||
[0,-0]
|
||||
(1 row)
|
||||
|
||||
SELECT '[4e38,1]'::minivec;
|
||||
ERROR: "4e38" is out of range for type minivec
|
||||
LINE 1: SELECT '[4e38,1]'::minivec;
|
||||
^
|
||||
SELECT '[1e-46,1]'::minivec;
|
||||
minivec
|
||||
---------
|
||||
[0,1]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[1,2,3"
|
||||
LINE 1: SELECT '[1,2,3'::minivec;
|
||||
^
|
||||
SELECT '[1,2,3]9'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[1,2,3]9"
|
||||
LINE 1: SELECT '[1,2,3]9'::minivec;
|
||||
^
|
||||
DETAIL: Junk after closing right brace.
|
||||
SELECT '1,2,3'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "1,2,3"
|
||||
LINE 1: SELECT '1,2,3'::minivec;
|
||||
^
|
||||
DETAIL: Vector contents must start with "[".
|
||||
SELECT ''::minivec;
|
||||
ERROR: invalid input syntax for type minivec: ""
|
||||
LINE 1: SELECT ''::minivec;
|
||||
^
|
||||
DETAIL: Vector contents must start with "[".
|
||||
SELECT '['::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "["
|
||||
LINE 1: SELECT '['::minivec;
|
||||
^
|
||||
SELECT '[ '::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[ "
|
||||
LINE 1: SELECT '[ '::minivec;
|
||||
^
|
||||
SELECT '[,'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[,"
|
||||
LINE 1: SELECT '[,'::minivec;
|
||||
^
|
||||
SELECT '[]'::minivec;
|
||||
ERROR: minivec must have at least 1 dimension
|
||||
LINE 1: SELECT '[]'::minivec;
|
||||
^
|
||||
SELECT '[ ]'::minivec;
|
||||
ERROR: minivec must have at least 1 dimension
|
||||
LINE 1: SELECT '[ ]'::minivec;
|
||||
^
|
||||
SELECT '[,]'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[,]"
|
||||
LINE 1: SELECT '[,]'::minivec;
|
||||
^
|
||||
SELECT '[1,]'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[1,]"
|
||||
LINE 1: SELECT '[1,]'::minivec;
|
||||
^
|
||||
SELECT '[1a]'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[1a]"
|
||||
LINE 1: SELECT '[1a]'::minivec;
|
||||
^
|
||||
SELECT '[1,,3]'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[1,,3]"
|
||||
LINE 1: SELECT '[1,,3]'::minivec;
|
||||
^
|
||||
SELECT '[1, ,3]'::minivec;
|
||||
ERROR: invalid input syntax for type minivec: "[1, ,3]"
|
||||
LINE 1: SELECT '[1, ,3]'::minivec;
|
||||
^
|
||||
SELECT '[1,2,3]'::minivec(3);
|
||||
minivec
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '[1,2,3]'::minivec(3, 2);
|
||||
ERROR: invalid type modifier
|
||||
LINE 1: SELECT '[1,2,3]'::minivec(3, 2);
|
||||
^
|
||||
SELECT '[1,2,3]'::minivec('a');
|
||||
ERROR: invalid input syntax for type integer: "a"
|
||||
LINE 1: SELECT '[1,2,3]'::minivec('a');
|
||||
^
|
||||
SELECT '[1,2,3]'::minivec(0);
|
||||
ERROR: dimensions for type minivec must be at least 1
|
||||
LINE 1: SELECT '[1,2,3]'::minivec(0);
|
||||
^
|
||||
SELECT '[1,2,3]'::minivec(16001);
|
||||
ERROR: dimensions for type minivec cannot exceed 16000
|
||||
LINE 1: SELECT '[1,2,3]'::minivec(16001);
|
||||
^
|
||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::minivec[]);
|
||||
unnest
|
||||
---------
|
||||
[1,2,3]
|
||||
[4,5,6]
|
||||
(2 rows)
|
||||
|
||||
SELECT '{"[1,2,3]"}'::minivec(2)[];
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT '[1,2,3]'::minivec + '[4,5,6]';
|
||||
?column?
|
||||
----------
|
||||
[5,7,8]
|
||||
(1 row)
|
||||
|
||||
SELECT '[61439]'::minivec + '[61439]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[1,2]'::minivec + '[3]';
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT '[1,2,3]'::minivec - '[4,5,6]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[-61439]'::minivec - '[61439]';
|
||||
?column?
|
||||
----------
|
||||
[-0]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::minivec - '[3]';
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT '[1,2,3]'::minivec * '[4,5,6]';
|
||||
?column?
|
||||
-----------
|
||||
[4,10,16]
|
||||
(1 row)
|
||||
|
||||
SELECT '[61439]'::minivec * '[61439]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[1e-7]'::minivec * '[1e-7]';
|
||||
?column?
|
||||
----------
|
||||
[0]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::minivec * '[3]';
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT '[1,2,3]'::minivec || '[4,5]';
|
||||
?column?
|
||||
-------------
|
||||
[1,2,3,4,5]
|
||||
(1 row)
|
||||
|
||||
SELECT array_fill(0, ARRAY[16000])::minivec || '[1]';
|
||||
ERROR: minivec cannot have more than 16000 dimensions
|
||||
SELECT '[1,2,3]'::minivec < '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec < '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec <= '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec <= '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec = '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec = '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec != '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec != '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec >= '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec >= '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec > '[1,2,3]';
|
||||
?column?
|
||||
----------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2,3]'::minivec > '[1,2]';
|
||||
?column?
|
||||
----------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[1,2,3]', '[1,2,3]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[1,2,3]', '[0,0,0]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[0,0,0]', '[1,2,3]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[1,2]', '[1,2,3]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[1,2,3]', '[1,2]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[1,2]', '[2,3,4]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
-1
|
||||
(1 row)
|
||||
|
||||
SELECT minivec_cmp('[2,3]', '[1,2,3]');
|
||||
minivec_cmp
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT vector_dims('[1,2,3]'::minivec);
|
||||
vector_dims
|
||||
-------------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT round(l2_norm('[1,1]'::minivec)::numeric, 5);
|
||||
round
|
||||
---------
|
||||
1.41421
|
||||
(1 row)
|
||||
|
||||
SELECT l2_norm('[3,4]'::minivec);
|
||||
l2_norm
|
||||
---------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT l2_norm('[0,1]'::minivec);
|
||||
l2_norm
|
||||
---------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l2_norm('[0,0]'::minivec);
|
||||
l2_norm
|
||||
---------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT l2_norm('[2]'::minivec);
|
||||
l2_norm
|
||||
---------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[0,0]'::minivec, '[3,4]');
|
||||
l2_distance
|
||||
-------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[0,0]'::minivec, '[0,1]');
|
||||
l2_distance
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[1,2]'::minivec, '[3]');
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,1,1,1,1,1,1,4,5]');
|
||||
l2_distance
|
||||
-------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,0]'::minivec <-> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]'::minivec, '[3,4]');
|
||||
inner_product
|
||||
---------------
|
||||
11
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]'::minivec, '[3]');
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT inner_product('[448]'::minivec, '[448]');
|
||||
inner_product
|
||||
---------------
|
||||
200704
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]');
|
||||
inner_product
|
||||
---------------
|
||||
44
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::minivec <#> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
-11
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::minivec, '[2,4]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::minivec, '[0,0]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[1,1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,0]'::minivec, '[0,2]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[-1,-1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::minivec, '[3]');
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[1.1,1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[-1.1,-1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT '[1,2]'::minivec <=> '[2,4]';
|
||||
?column?
|
||||
----------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[0,0]'::minivec, '[3,4]');
|
||||
l1_distance
|
||||
-------------
|
||||
7
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[0,0]'::minivec, '[0,1]');
|
||||
l1_distance
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[1,2]'::minivec, '[3]');
|
||||
ERROR: different minivec dimensions 2 and 1
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
|
||||
l1_distance
|
||||
-------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]');
|
||||
l1_distance
|
||||
-------------
|
||||
7
|
||||
(1 row)
|
||||
|
||||
SELECT '[0,0]'::minivec <+> '[3,4]';
|
||||
?column?
|
||||
----------
|
||||
7
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[3,4]'::minivec);
|
||||
l2_normalize
|
||||
--------------
|
||||
[0.625,0.75]
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[3,0]'::minivec);
|
||||
l2_normalize
|
||||
--------------
|
||||
[1,0]
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[0,0.1]'::minivec);
|
||||
l2_normalize
|
||||
--------------
|
||||
[0,1]
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[0,0]'::minivec);
|
||||
l2_normalize
|
||||
--------------
|
||||
[0,0]
|
||||
(1 row)
|
||||
|
||||
SELECT l2_normalize('[448]'::minivec);
|
||||
l2_normalize
|
||||
--------------
|
||||
[1]
|
||||
(1 row)
|
||||
|
||||
SELECT binary_quantize('[1,0,-1]'::minivec);
|
||||
binary_quantize
|
||||
-----------------
|
||||
100
|
||||
(1 row)
|
||||
|
||||
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::minivec);
|
||||
binary_quantize
|
||||
-----------------
|
||||
01001110101
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 1, 3);
|
||||
subvector
|
||||
-----------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2);
|
||||
subvector
|
||||
-----------
|
||||
[3,4]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, -1, 3);
|
||||
subvector
|
||||
-----------
|
||||
[1]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, 9);
|
||||
subvector
|
||||
-----------
|
||||
[3,4,5]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 1, 0);
|
||||
ERROR: minivec must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, -1);
|
||||
ERROR: minivec must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, -1, 2);
|
||||
ERROR: minivec must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 2147483647, 10);
|
||||
ERROR: minivec must have at least 1 dimension
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2147483647);
|
||||
subvector
|
||||
-----------
|
||||
[3,4,5]
|
||||
(1 row)
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, -2147483644, 2147483647);
|
||||
subvector
|
||||
-----------
|
||||
[1,2]
|
||||
(1 row)
|
||||
|
||||
@@ -22,17 +22,6 @@ SELECT * FROM t ORDER BY val;
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
-- minivec
|
||||
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t (val);
|
||||
|
||||
SELECT * FROM t WHERE val = '[1,2,3]';
|
||||
SELECT * FROM t ORDER BY val;
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
-- sparsevec
|
||||
|
||||
CREATE TABLE t (val sparsevec(3));
|
||||
|
||||
@@ -38,22 +38,6 @@ SELECT '{1,2,3}'::real[]::halfvec(2);
|
||||
SELECT '{65520,-65520}'::real[]::halfvec;
|
||||
SELECT '{1e-8,-1e-8}'::real[]::halfvec;
|
||||
|
||||
SELECT '[1,2,3]'::vector::minivec;
|
||||
SELECT '[1,2,3]'::vector::minivec(3);
|
||||
SELECT '[1,2,3]'::vector::minivec(2);
|
||||
SELECT '[61440]'::vector::minivec;
|
||||
SELECT '[1e-8]'::vector::minivec;
|
||||
|
||||
SELECT '[1,2,3]'::minivec::vector;
|
||||
SELECT '[1,2,3]'::minivec::vector(3);
|
||||
SELECT '[1,2,3]'::minivec::vector(2);
|
||||
|
||||
SELECT '{1,2,3}'::real[]::minivec;
|
||||
SELECT '{1,2,3}'::real[]::minivec(3);
|
||||
SELECT '{1,2,3}'::real[]::minivec(2);
|
||||
SELECT '{61440,-61440}'::real[]::minivec;
|
||||
SELECT '{1e-8,-1e-8}'::real[]::minivec;
|
||||
|
||||
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
|
||||
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(5);
|
||||
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(4);
|
||||
|
||||
@@ -28,21 +28,6 @@ SELECT * FROM t2 ORDER BY val;
|
||||
DROP TABLE t;
|
||||
DROP TABLE t2;
|
||||
|
||||
-- minivec
|
||||
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
|
||||
CREATE TABLE t2 (val minivec(3));
|
||||
|
||||
\copy t TO 'results/minivec.bin' WITH (FORMAT binary)
|
||||
\copy t2 FROM 'results/minivec.bin' WITH (FORMAT binary)
|
||||
|
||||
SELECT * FROM t2 ORDER BY val;
|
||||
|
||||
DROP TABLE t;
|
||||
DROP TABLE t2;
|
||||
|
||||
-- sparsevec
|
||||
|
||||
CREATE TABLE t (val sparsevec(3));
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
-- L2
|
||||
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val minivec_l2_ops) WITH (lists = 1);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::minivec)) t2;
|
||||
SELECT COUNT(*) FROM t;
|
||||
|
||||
TRUNCATE t;
|
||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
-- inner product
|
||||
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val minivec_ip_ops) WITH (lists = 1);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::minivec)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
-- cosine
|
||||
|
||||
CREATE TABLE t (val minivec(3));
|
||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val minivec_cosine_ops) WITH (lists = 1);
|
||||
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::minivec)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,134 +0,0 @@
|
||||
SELECT '[1,2,3]'::minivec;
|
||||
SELECT '[-1,-2,-3]'::minivec;
|
||||
SELECT '[1.,2.,3.]'::minivec;
|
||||
SELECT ' [ 1, 2 , 3 ] '::minivec;
|
||||
SELECT '[1.23456]'::minivec;
|
||||
SELECT '[hello,1]'::minivec;
|
||||
SELECT '[NaN,1]'::minivec;
|
||||
SELECT '[Infinity,1]'::minivec;
|
||||
SELECT '[-Infinity,1]'::minivec;
|
||||
SELECT '[61439,-61439]'::minivec;
|
||||
SELECT '[61440,-61440]'::minivec;
|
||||
SELECT '[1e-8,-1e-8]'::minivec;
|
||||
SELECT '[4e38,1]'::minivec;
|
||||
SELECT '[1e-46,1]'::minivec;
|
||||
SELECT '[1,2,3'::minivec;
|
||||
SELECT '[1,2,3]9'::minivec;
|
||||
SELECT '1,2,3'::minivec;
|
||||
SELECT ''::minivec;
|
||||
SELECT '['::minivec;
|
||||
SELECT '[ '::minivec;
|
||||
SELECT '[,'::minivec;
|
||||
SELECT '[]'::minivec;
|
||||
SELECT '[ ]'::minivec;
|
||||
SELECT '[,]'::minivec;
|
||||
SELECT '[1,]'::minivec;
|
||||
SELECT '[1a]'::minivec;
|
||||
SELECT '[1,,3]'::minivec;
|
||||
SELECT '[1, ,3]'::minivec;
|
||||
|
||||
SELECT '[1,2,3]'::minivec(3);
|
||||
SELECT '[1,2,3]'::minivec(2);
|
||||
SELECT '[1,2,3]'::minivec(3, 2);
|
||||
SELECT '[1,2,3]'::minivec('a');
|
||||
SELECT '[1,2,3]'::minivec(0);
|
||||
SELECT '[1,2,3]'::minivec(16001);
|
||||
|
||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::minivec[]);
|
||||
SELECT '{"[1,2,3]"}'::minivec(2)[];
|
||||
|
||||
SELECT '[1,2,3]'::minivec + '[4,5,6]';
|
||||
SELECT '[61439]'::minivec + '[61439]';
|
||||
SELECT '[1,2]'::minivec + '[3]';
|
||||
|
||||
SELECT '[1,2,3]'::minivec - '[4,5,6]';
|
||||
SELECT '[-61439]'::minivec - '[61439]';
|
||||
SELECT '[1,2]'::minivec - '[3]';
|
||||
|
||||
SELECT '[1,2,3]'::minivec * '[4,5,6]';
|
||||
SELECT '[61439]'::minivec * '[61439]';
|
||||
SELECT '[1e-7]'::minivec * '[1e-7]';
|
||||
SELECT '[1,2]'::minivec * '[3]';
|
||||
|
||||
SELECT '[1,2,3]'::minivec || '[4,5]';
|
||||
SELECT array_fill(0, ARRAY[16000])::minivec || '[1]';
|
||||
|
||||
SELECT '[1,2,3]'::minivec < '[1,2,3]';
|
||||
SELECT '[1,2,3]'::minivec < '[1,2]';
|
||||
SELECT '[1,2,3]'::minivec <= '[1,2,3]';
|
||||
SELECT '[1,2,3]'::minivec <= '[1,2]';
|
||||
SELECT '[1,2,3]'::minivec = '[1,2,3]';
|
||||
SELECT '[1,2,3]'::minivec = '[1,2]';
|
||||
SELECT '[1,2,3]'::minivec != '[1,2,3]';
|
||||
SELECT '[1,2,3]'::minivec != '[1,2]';
|
||||
SELECT '[1,2,3]'::minivec >= '[1,2,3]';
|
||||
SELECT '[1,2,3]'::minivec >= '[1,2]';
|
||||
SELECT '[1,2,3]'::minivec > '[1,2,3]';
|
||||
SELECT '[1,2,3]'::minivec > '[1,2]';
|
||||
|
||||
SELECT minivec_cmp('[1,2,3]', '[1,2,3]');
|
||||
SELECT minivec_cmp('[1,2,3]', '[0,0,0]');
|
||||
SELECT minivec_cmp('[0,0,0]', '[1,2,3]');
|
||||
SELECT minivec_cmp('[1,2]', '[1,2,3]');
|
||||
SELECT minivec_cmp('[1,2,3]', '[1,2]');
|
||||
SELECT minivec_cmp('[1,2]', '[2,3,4]');
|
||||
SELECT minivec_cmp('[2,3]', '[1,2,3]');
|
||||
|
||||
SELECT vector_dims('[1,2,3]'::minivec);
|
||||
|
||||
SELECT round(l2_norm('[1,1]'::minivec)::numeric, 5);
|
||||
SELECT l2_norm('[3,4]'::minivec);
|
||||
SELECT l2_norm('[0,1]'::minivec);
|
||||
SELECT l2_norm('[0,0]'::minivec);
|
||||
SELECT l2_norm('[2]'::minivec);
|
||||
|
||||
SELECT l2_distance('[0,0]'::minivec, '[3,4]');
|
||||
SELECT l2_distance('[0,0]'::minivec, '[0,1]');
|
||||
SELECT l2_distance('[1,2]'::minivec, '[3]');
|
||||
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,1,1,1,1,1,1,4,5]');
|
||||
SELECT '[0,0]'::minivec <-> '[3,4]';
|
||||
|
||||
SELECT inner_product('[1,2]'::minivec, '[3,4]');
|
||||
SELECT inner_product('[1,2]'::minivec, '[3]');
|
||||
SELECT inner_product('[448]'::minivec, '[448]');
|
||||
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::minivec, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT '[1,2]'::minivec <#> '[3,4]';
|
||||
|
||||
SELECT cosine_distance('[1,2]'::minivec, '[2,4]');
|
||||
SELECT cosine_distance('[1,2]'::minivec, '[0,0]');
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[1,1]');
|
||||
SELECT cosine_distance('[1,0]'::minivec, '[0,2]');
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[-1,-1]');
|
||||
SELECT cosine_distance('[1,2]'::minivec, '[3]');
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]'::minivec, '[-1.1,-1.1]');
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
|
||||
SELECT '[1,2]'::minivec <=> '[2,4]';
|
||||
|
||||
SELECT l1_distance('[0,0]'::minivec, '[3,4]');
|
||||
SELECT l1_distance('[0,0]'::minivec, '[0,1]');
|
||||
SELECT l1_distance('[1,2]'::minivec, '[3]');
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[1,2,3,4,5,6,7,8,9]');
|
||||
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::minivec, '[0,3,2,5,4,7,6,9,8]');
|
||||
SELECT '[0,0]'::minivec <+> '[3,4]';
|
||||
|
||||
SELECT l2_normalize('[3,4]'::minivec);
|
||||
SELECT l2_normalize('[3,0]'::minivec);
|
||||
SELECT l2_normalize('[0,0.1]'::minivec);
|
||||
SELECT l2_normalize('[0,0]'::minivec);
|
||||
SELECT l2_normalize('[448]'::minivec);
|
||||
|
||||
SELECT binary_quantize('[1,0,-1]'::minivec);
|
||||
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::minivec);
|
||||
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, -1, 3);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, 9);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 1, 0);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, -1);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, -1, 2);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 2147483647, 10);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, 3, 2147483647);
|
||||
SELECT subvector('[1,2,3,4,5]'::minivec, -2147483644, 2147483647);
|
||||
@@ -94,8 +94,7 @@ like($explain, qr/Seq Scan/);
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query';
|
||||
));
|
||||
# TODO Do not use index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
like($explain, qr/Seq Scan/);
|
||||
|
||||
# Test attribute index
|
||||
$node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
|
||||
@@ -110,7 +109,6 @@ $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING ivfflat (v v
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
# TODO Use partial index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
like($explain, qr/Index Scan using partial_idx/);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -18,9 +18,13 @@ $node->start;
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);");
|
||||
$node->safe_psql("postgres", "CREATE TABLE cat (i int4 PRIMARY KEY, t text, b boolean);");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO cat SELECT i, 'cat ' || i, i % 5 = 0 FROM generate_series(1, $nc) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
@@ -96,13 +100,25 @@ $explain = $node->safe_psql("postgres", qq(
|
||||
));
|
||||
like($explain, qr/Seq Scan/);
|
||||
|
||||
# Test join
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test join with attribute filtering
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c WHERE cat.b = 't' ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
# Test attribute index
|
||||
$node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
|
||||
$explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
# TODO Use attribute index
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
# Use attribute index
|
||||
like($explain, qr/Bitmap Index Scan on attribute_idx/);
|
||||
|
||||
# Test partial index
|
||||
$node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);");
|
||||
|
||||
@@ -40,10 +40,6 @@ for (1 .. 50)
|
||||
$actual = $node->safe_psql("postgres", "SELECT halfvec_cmp(v::halfvec, '$query'::real[]::halfvec) FROM tst");
|
||||
is($expected, $actual);
|
||||
|
||||
# Test minivec
|
||||
$actual = $node->safe_psql("postgres", "SELECT minivec_cmp(v::minivec, '$query'::real[]::minivec) FROM tst");
|
||||
is($expected, $actual);
|
||||
|
||||
# Test sparsevec
|
||||
$actual = $node->safe_psql("postgres", "SELECT sparsevec_cmp(v::vector::sparsevec, '$query'::real[]::vector::sparsevec) FROM tst");
|
||||
is($expected, $actual);
|
||||
|
||||
@@ -45,10 +45,6 @@ for my $function (@functions)
|
||||
my $actual = $node->safe_psql("postgres", "SELECT $function(v::halfvec, '$query'::vector::halfvec) FROM tst");
|
||||
is($expected, $actual, "halfvec $function");
|
||||
|
||||
# Test minivec
|
||||
$actual = $node->safe_psql("postgres", "SELECT $function(v::minivec, '$query'::vector::minivec) FROM tst");
|
||||
is($expected, $actual, "minivec $function");
|
||||
|
||||
# Test sparsevec
|
||||
$actual = $node->safe_psql("postgres", "SELECT $function(v::sparsevec, '$query'::vector::sparsevec) FROM tst");
|
||||
is($expected, $actual, "sparsevec $function");
|
||||
|
||||
@@ -12,8 +12,8 @@ $node->start;
|
||||
# Create extension
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
|
||||
my @types = ("vector", "halfvec", "minivec", "sparsevec");
|
||||
my @inputs = ("[1.23,4.56,7.89]", "[1.23,4.56,7.89]", "[1.23,4.56,7.89]", "{1:1.23,2:4.56,3:7.89}/3");
|
||||
my @types = ("vector", "halfvec", "sparsevec");
|
||||
my @inputs = ("[1.23,4.56,7.89]", "[1.23,4.56,7.89]", "{1:1.23,2:4.56,3:7.89}/3");
|
||||
my @subs = (" ", " ", ",", ":", "-", "1", "9", "\0", "2147483648", "-2147483649");
|
||||
|
||||
for my $i (0 .. $#types)
|
||||
|
||||
46
test/t/039_hnsw_cost.pl
Normal file
46
test/t/039_hnsw_cost.pl
Normal file
@@ -0,0 +1,46 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my @dims = (384, 1536);
|
||||
my $limit = 10;
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
|
||||
for my $dim (@dims)
|
||||
{
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
my $n = $dim == 384 ? 2000 : 1000;
|
||||
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, $n) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
# Generate query
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
my $query = "[" . join(",", @r) . "]";
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP TABLE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,132 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 10;
|
||||
my $array_sql = join(",", ('2 * random() * random()') x $dim);
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v minivec($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
push(@queries, "[" . join(",", @r) . "]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("minivec_l2_ops", "minivec_ip_ops", "minivec_cosine_ops", "minivec_l1_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Build index serially
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET max_parallel_maintenance_workers = 0;
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
|
||||
# Test approximate results
|
||||
my $min = 0.98;
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel in memory
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET client_min_messages = DEBUG;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
|
||||
# Test approximate results
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel on disk
|
||||
# Set parallel_workers on table to use workers with low maintenance_work_mem
|
||||
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
ALTER TABLE tst SET (parallel_workers = 2);
|
||||
SET client_min_messages = DEBUG;
|
||||
SET maintenance_work_mem = '4MB';
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
ALTER TABLE tst RESET (parallel_workers);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,113 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 10;
|
||||
my $array_sql = join(",", ('2 * random() * random()') x $dim);
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v minivec($dim));");
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
push(@queries, "[" . join(",", @r) . "]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>", "<+>");
|
||||
my @opclasses = ("minivec_l2_ops", "minivec_ip_ops", "minivec_cosine_ops", "minivec_l1_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Add index
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v $opclass);");
|
||||
|
||||
# Use concurrent inserts
|
||||
$node->pgbench(
|
||||
"--no-vacuum --client=10 --transactions=1000",
|
||||
0,
|
||||
[qr{actually processed}],
|
||||
[qr{^$}],
|
||||
"concurrent INSERTs",
|
||||
{
|
||||
"040_hnsw_minivec_insert_recall_$opclass" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);"
|
||||
}
|
||||
);
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_indexscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Test approximate results
|
||||
my $min = 0.98;
|
||||
test_recall($min, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
$node->safe_psql("postgres", "TRUNCATE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
45
test/t/040_ivfflat_cost.pl
Normal file
45
test/t/040_ivfflat_cost.pl
Normal file
@@ -0,0 +1,45 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my @dims = (384, 1536);
|
||||
my $limit = 10;
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
|
||||
for my $dim (@dims)
|
||||
{
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 5000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5);");
|
||||
$node->safe_psql("postgres", "ANALYZE tst;");
|
||||
|
||||
# Generate query
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
my $query = "[" . join(",", @r) . "]";
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx/);
|
||||
|
||||
$node->safe_psql("postgres", "DROP TABLE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,97 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($min, $ef_search, $test_name) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = $ef_search;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = $ef_search;
|
||||
SELECT i FROM tst ORDER BY v <-> '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $test_name);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v minivec(3));");
|
||||
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
|
||||
# Add index
|
||||
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v minivec_l2_ops) WITH (m = 4, ef_construction = 8);");
|
||||
|
||||
# Delete data
|
||||
$node->safe_psql("postgres", "DELETE FROM tst WHERE i > 2500;");
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r1 = rand();
|
||||
my $r2 = rand();
|
||||
my $r3 = rand();
|
||||
push(@queries, "[$r1,$r2,$r3]");
|
||||
}
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_indexscan = off;
|
||||
SELECT i FROM tst ORDER BY v <-> '$_' LIMIT $limit;
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
test_recall(0.18, $limit, "before vacuum");
|
||||
test_recall(0.93, 100, "before vacuum");
|
||||
|
||||
# TODO Test concurrent inserts with vacuum
|
||||
$node->safe_psql("postgres", "VACUUM tst;");
|
||||
|
||||
test_recall(0.95, $limit, "after vacuum");
|
||||
|
||||
done_testing();
|
||||
66
test/t/041_hnsw_streaming.pl
Normal file
66
test/t/041_hnsw_streaming.pl
Normal file
@@ -0,0 +1,66 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $dim = 3;
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4 PRIMARY KEY, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET maintenance_work_mem = '128MB';
|
||||
SET max_parallel_maintenance_workers = 2;
|
||||
CREATE INDEX ON tst USING hnsw (v vector_l2_ops)
|
||||
));
|
||||
|
||||
my $count = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.streaming = on;
|
||||
SET work_mem = '8MB';
|
||||
SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
|
||||
));
|
||||
is($count, 10);
|
||||
|
||||
foreach ((30000, 50000, 70000))
|
||||
{
|
||||
my $ef_stream = $_;
|
||||
my $expected = $ef_stream / 10000;
|
||||
my $sum = 0;
|
||||
|
||||
for my $i (1 .. 20)
|
||||
{
|
||||
$count = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.streaming = on;
|
||||
SET hnsw.ef_stream = $ef_stream;
|
||||
SET work_mem = '8MB';
|
||||
SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst WHERE i = $i) LIMIT 11) t;
|
||||
));
|
||||
$sum += $count;
|
||||
}
|
||||
|
||||
my $avg = $sum / 20;
|
||||
cmp_ok($avg, '>', $expected - 2);
|
||||
cmp_ok($avg, '<', $expected + 2);
|
||||
}
|
||||
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.streaming = on;
|
||||
SET work_mem = '2MB';
|
||||
SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;
|
||||
));
|
||||
like($stderr, qr/hnsw index scan exceeded work_mem after \d+ tuples/);
|
||||
|
||||
done_testing();
|
||||
@@ -1,58 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (v minivec(3));");
|
||||
|
||||
sub insert_vectors
|
||||
{
|
||||
for my $i (1 .. 20)
|
||||
{
|
||||
$node->safe_psql("postgres", "INSERT INTO tst VALUES ('[1,1,1]');");
|
||||
}
|
||||
}
|
||||
|
||||
sub test_duplicates
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = 1;
|
||||
SELECT COUNT(*) FROM (SELECT * FROM tst ORDER BY v <-> '[1,1,1]') t;
|
||||
));
|
||||
is($res, 10);
|
||||
}
|
||||
|
||||
# Test duplicates with build
|
||||
insert_vectors();
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v minivec_l2_ops);");
|
||||
test_duplicates();
|
||||
|
||||
# Reset
|
||||
$node->safe_psql("postgres", "TRUNCATE tst;");
|
||||
|
||||
# Test duplicates with inserts
|
||||
insert_vectors();
|
||||
test_duplicates();
|
||||
|
||||
# Test fallback path for inserts
|
||||
$node->pgbench(
|
||||
"--no-vacuum --client=5 --transactions=100",
|
||||
0,
|
||||
[qr{actually processed}],
|
||||
[qr{^$}],
|
||||
"concurrent INSERTs",
|
||||
{
|
||||
"042_hnsw_minivec_duplicates" => "INSERT INTO tst VALUES ('[1,1,1]');"
|
||||
}
|
||||
);
|
||||
|
||||
done_testing();
|
||||
131
test/t/042_hnsw_streaming_recall.pl
Normal file
131
test/t/042_hnsw_streaming_recall.pl
Normal file
@@ -0,0 +1,131 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 3;
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
my @cs = (100, 1000);
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($c, $ef_search, $min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = $ef_search;
|
||||
SET hnsw.streaming = on;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx on tst/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET hnsw.ef_search = $ef_search;
|
||||
SET hnsw.streaming = on;
|
||||
SELECT i FROM tst WHERE i % $c = 0 ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
my %expected_set = map { $_ => 1 } @expected_ids;
|
||||
|
||||
foreach (@actual_ids)
|
||||
{
|
||||
if (exists($expected_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
}
|
||||
|
||||
$total += $limit;
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
|
||||
);
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
push(@queries, "[" . join(",", @r) . "]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<=>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET maintenance_work_mem = '128MB';
|
||||
CREATE INDEX idx ON tst USING hnsw (v $opclass);
|
||||
));
|
||||
|
||||
foreach (@cs)
|
||||
{
|
||||
my $c = $_;
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_indexscan = off;
|
||||
WITH top AS (
|
||||
SELECT v $operator '$_' AS distance FROM tst WHERE i % $c = 0 ORDER BY distance LIMIT $limit
|
||||
)
|
||||
SELECT i FROM tst WHERE (v $operator '$_') <= (SELECT MAX(distance) FROM top)
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
if ($c == 100)
|
||||
{
|
||||
test_recall($c, 40, 0.99, $operator);
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($operator eq "<->")
|
||||
{
|
||||
test_recall($c, 40, 0.99, $operator);
|
||||
}
|
||||
else
|
||||
{
|
||||
test_recall($c, 40, 0.99, $operator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,154 +0,0 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 10;
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($probes, $min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx on tst/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
my %expected_set = map { $_ => 1 } @expected_ids;
|
||||
|
||||
foreach (@actual_ids)
|
||||
{
|
||||
if (exists($expected_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
}
|
||||
|
||||
$total += $limit;
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v minivec($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
|
||||
);
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my @r = ();
|
||||
for (1 .. $dim)
|
||||
{
|
||||
push(@r, rand());
|
||||
}
|
||||
push(@queries, "[" . join(",", @r) . "]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("minivec_l2_ops", "minivec_ip_ops", "minivec_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
WITH top AS (
|
||||
SELECT v $operator '$_' AS distance FROM tst ORDER BY distance LIMIT $limit
|
||||
)
|
||||
SELECT i FROM tst WHERE (v $operator '$_') <= (SELECT MAX(distance) FROM top)
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Build index serially
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET max_parallel_maintenance_workers = 0;
|
||||
CREATE INDEX idx ON tst USING ivfflat (v $opclass);
|
||||
));
|
||||
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
test_recall(1, 0.33, $operator);
|
||||
test_recall(10, 0.93, $operator);
|
||||
}
|
||||
|
||||
# Test probes equals lists
|
||||
if ($operator eq "<=>")
|
||||
{
|
||||
test_recall(100, 0.98, $operator);
|
||||
}
|
||||
else
|
||||
{
|
||||
test_recall(100, 1.00, $operator);
|
||||
}
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET client_min_messages = DEBUG;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
CREATE INDEX idx ON tst USING ivfflat (v $opclass);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
test_recall(1, 0.33, $operator);
|
||||
test_recall(10, 0.93, $operator);
|
||||
}
|
||||
|
||||
# Test probes equals lists
|
||||
if ($operator eq "<=>")
|
||||
{
|
||||
test_recall(100, 0.98, $operator);
|
||||
}
|
||||
else
|
||||
{
|
||||
test_recall(100, 1.00, $operator);
|
||||
}
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
Reference in New Issue
Block a user