Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
4f4286f74e Updated handling of range error to be consistent with real 2024-03-30 08:59:23 -07:00
26 changed files with 147 additions and 880 deletions

View File

@@ -40,21 +40,13 @@ jobs:
sudo apt-get install libipc-run-perl sudo apt-get install libipc-run-perl
- run: make prove_installcheck - run: make prove_installcheck
mac: mac:
runs-on: ${{ matrix.os }} runs-on: macos-latest
if: ${{ !startsWith(github.ref_name, 'windows') }} if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 16
os: macos-14
- postgres: 14
os: macos-12
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: 14
- run: make - run: make
env: env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
@@ -62,19 +54,13 @@ jobs:
- run: make installcheck - run: make installcheck
- if: ${{ failure() }} - if: ${{ failure() }}
run: cat regression.diffs run: cat regression.diffs
# Homebrew Postgres does not enable TAP tests, so need to download
- run: | - run: |
brew install cpanm brew install cpanm
cpanm --notest IPC::Run cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
tar xf $TAG.tar.gz tar xf REL_14_10.tar.gz
mv postgres-$TAG postgres - run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
env: - run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
env: env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows: windows:

View File

@@ -1,13 +1,3 @@
## 0.7.0 (unreleased)
- Added support for bit vectors to HNSW
- Added `hamming_distance` function
- Added `jaccard_distance` function
- Added `quantize_binary` function
- Added `subvector` function
- Added concatenate operator for vectors
- Updated comparison operators to support vectors with different dimensions
## 0.6.2 (2024-03-18) ## 0.6.2 (2024-03-18)
- Reduced lock contention with parallel HNSW index builds - Reduced lock contention with parallel HNSW index builds

View File

@@ -3,7 +3,7 @@ EXTVERSION = 0.6.2
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*.sql) DATA = $(wildcard sql/*--*.sql)
OBJS = src/bitvector.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/vector.o OBJS = src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/vector.o
HEADERS = src/vector.h HEADERS = src/vector.h
TESTS = $(wildcard test/sql/*.sql) TESTS = $(wildcard test/sql/*.sql)
@@ -12,7 +12,7 @@ REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
OPTFLAGS = -march=native OPTFLAGS = -march=native
# Mac ARM doesn't always support -march=native # Mac ARM doesn't support -march=native
ifeq ($(shell uname -s), Darwin) ifeq ($(shell uname -s), Darwin)
ifeq ($(shell uname -p), arm) ifeq ($(shell uname -p), arm)
# no difference with -march=armv8.5-a # no difference with -march=armv8.5-a

View File

@@ -1,10 +1,10 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.6.2 EXTVERSION = 0.6.2
OBJS = src\bitvector.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
HEADERS = src\vector.h HEADERS = src\vector.h
REGRESS = bit_functions btree cast copy hnsw_cosine hnsw_hamming hnsw_ip hnsw_jaccard hnsw_l2 hnsw_options hnsw_unlogged ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged vector_functions vector_input REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION) REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# For /arch flags # For /arch flags

View File

@@ -221,19 +221,7 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops); CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
``` ```
Hamming distance - unreleased Vectors with up to 2,000 dimensions can be indexed.
```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
```
Jaccard distance - unreleased
```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
```
Vectors with up to 2,000 dimensions can be indexed, or bit vectors with up to 64,000 dimensions.
### Index Options ### Index Options
@@ -711,9 +699,6 @@ Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosi
## Reference ## Reference
- [Vector](#vector-type)
- [Bit](#bit-type)
### Vector Type ### Vector Type
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single-precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions. Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single-precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
@@ -725,7 +710,6 @@ Operator | Description | Added
\+ | element-wise addition | \+ | element-wise addition |
\- | element-wise subtraction | \- | element-wise subtraction |
\* | element-wise multiplication | 0.5.0 \* | element-wise multiplication | 0.5.0
\|\| | concatenate | unreleased
<-> | Euclidean distance | <-> | Euclidean distance |
<#> | negative inner product | <#> | negative inner product |
<=> | cosine distance | <=> | cosine distance |
@@ -738,8 +722,6 @@ cosine_distance(vector, vector) → double precision | cosine distance |
inner_product(vector, vector) → double precision | inner product | inner_product(vector, vector) → double precision | inner product |
l2_distance(vector, vector) → double precision | Euclidean distance | l2_distance(vector, vector) → double precision | Euclidean distance |
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0 l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
quantize_binary(vector) → bit | quantize | unreleased
subvector(vector, integer, integer) → vector | subvector | unreleased
vector_dims(vector) → integer | number of dimensions | vector_dims(vector) → integer | number of dimensions |
vector_norm(vector) → double precision | Euclidean norm | vector_norm(vector) → double precision | Euclidean norm |
@@ -750,24 +732,6 @@ Function | Description | Added
avg(vector) → vector | average | avg(vector) → vector | average |
sum(vector) → vector | sum | 0.5.0 sum(vector) → vector | sum | 0.5.0
### Bit Type
Each bit vector takes `dimensions / 8 + (5 or 8)` bytes of storage. See the [Postgres docs](https://www.postgresql.org/docs/current/datatype-bit.html) for more info.
### Bit Operators
Operator | Description | Added
--- | --- | ---
<~> | Hamming distance | unreleased
<%> | Jaccard distance | unreleased
### Bit Functions
Function | Description | Added
--- | --- | ---
hamming_distance(bit, bit) → double precision | Hamming distance | unreleased
jaccard_distance(bit, bit) → double precision | Jaccard distance | unreleased
## Installation Notes - Linux and Mac ## Installation Notes - Linux and Mac
### Postgres Location ### Postgres Location

View File

@@ -1,41 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.0'" to load this file. \quit
CREATE FUNCTION quantize_binary(vector) RETURNS bit
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_concat(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR || (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_concat
);
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit);

View File

@@ -58,12 +58,6 @@ CREATE FUNCTION vector_sub(vector, vector) RETURNS vector
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION quantize_binary(vector) RETURNS bit
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector private functions -- vector private functions
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
@@ -105,9 +99,6 @@ CREATE FUNCTION vector_avg(double precision[]) RETURNS vector
CREATE FUNCTION vector_combine(double precision[], double precision[]) RETURNS double precision[] CREATE FUNCTION vector_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_concat(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector aggregates -- vector aggregates
CREATE AGGREGATE avg(vector) ( CREATE AGGREGATE avg(vector) (
@@ -197,10 +188,6 @@ CREATE OPERATOR * (
COMMUTATOR = * COMMUTATOR = *
); );
CREATE OPERATOR || (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_concat
);
CREATE OPERATOR < ( CREATE OPERATOR < (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_lt, LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_lt,
COMMUTATOR = > , NEGATOR = >= , COMMUTATOR = > , NEGATOR = >= ,
@@ -300,31 +287,3 @@ CREATE OPERATOR CLASS vector_cosine_ops
OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops, OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 vector_negative_inner_product(vector, vector), FUNCTION 1 vector_negative_inner_product(vector, vector),
FUNCTION 2 vector_norm(vector); FUNCTION 2 vector_norm(vector);
-- bit functions
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit);

View File

@@ -1,90 +0,0 @@
#include "postgres.h"
#include "bitvector.h"
#include "port/pg_bitutils.h"
#include "utils/varbit.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate and initialize a new bit vector
*/
VarBit *
InitBitVector(int dim)
{
VarBit *result;
int size;
size = VARBITTOTALLEN(dim);
result = (VarBit *) palloc0(size);
SET_VARSIZE(result, size);
VARBITLEN(result) = dim;
return result;
}
/*
* Ensure same dimensions
*/
static inline void
CheckDims(VarBit *a, VarBit *b)
{
if (VARBITLEN(a) != VARBITLEN(b))
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("different bit lengths %u and %u", VARBITLEN(a), VARBITLEN(b))));
}
/*
* Get the Hamming distance between two bit vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
Datum
hamming_distance(PG_FUNCTION_ARGS)
{
VarBit *a = PG_GETARG_VARBIT_P(0);
VarBit *b = PG_GETARG_VARBIT_P(1);
unsigned char *ax = VARBITS(a);
unsigned char *bx = VARBITS(b);
uint64 distance = 0;
CheckDims(a, b);
/* TODO Improve performance */
for (uint32 i = 0; i < VARBITBYTES(a); i++)
distance += pg_number_of_ones[ax[i] ^ bx[i]];
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the Jaccard distance between two bit vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(jaccard_distance);
Datum
jaccard_distance(PG_FUNCTION_ARGS)
{
VarBit *a = PG_GETARG_VARBIT_P(0);
VarBit *b = PG_GETARG_VARBIT_P(1);
unsigned char *ax = VARBITS(a);
unsigned char *bx = VARBITS(b);
uint64 ab = 0;
uint64 aa;
uint64 bb;
CheckDims(a, b);
/* TODO Improve performance */
for (uint32 i = 0; i < VARBITBYTES(a); i++)
ab += pg_number_of_ones[ax[i] & bx[i]];
if (ab == 0)
PG_RETURN_FLOAT8(1);
aa = pg_popcount((char *) ax, VARBITBYTES(a));
bb = pg_popcount((char *) bx, VARBITBYTES(b));
PG_RETURN_FLOAT8(1 - (ab / ((double) (aa + bb - ab))));
}

View File

@@ -1,8 +0,0 @@
#ifndef BITVECTOR_H
#define BITVECTOR_H
#include "utils/varbit.h"
VarBit *InitBitVector(int dim);
#endif

View File

@@ -57,8 +57,7 @@
typedef enum HnswType typedef enum HnswType
{ {
HNSW_TYPE_VECTOR, HNSW_TYPE_VECTOR
HNSW_TYPE_BIT
} HnswType; } HnswType;
/* Build phases */ /* Build phases */

View File

@@ -44,7 +44,6 @@
#include "access/xact.h" #include "access/xact.h"
#include "access/xloginsert.h" #include "access/xloginsert.h"
#include "catalog/index.h" #include "catalog/index.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h" #include "commands/progress.h"
#include "hnsw.h" #include "hnsw.h"
#include "miscadmin.h" #include "miscadmin.h"
@@ -666,27 +665,13 @@ HnswSharedMemoryAlloc(Size size, void *state)
return chunk; return chunk;
} }
/*
* Get max dimensions
*/
static int
GetMaxDimensions(HnswType type)
{
int maxDimensions = HNSW_MAX_DIM;
if (type == HNSW_TYPE_BIT)
maxDimensions *= 32;
return maxDimensions;
}
/* /*
* Initialize the build state * Initialize the build state
*/ */
static void static void
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum) InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
{ {
int maxDimensions; int maxDimensions = HNSW_MAX_DIM;
buildstate->heap = heap; buildstate->heap = heap;
buildstate->index = index; buildstate->index = index;
@@ -698,8 +683,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
buildstate->efConstruction = HnswGetEfConstruction(index); buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod; buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
maxDimensions = GetMaxDimensions(buildstate->type);
/* Require column to have dimensions to be indexed */ /* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0) if (buildstate->dimensions < 0)
elog(ERROR, "column does not have dimensions"); elog(ERROR, "column does not have dimensions");

View File

@@ -1,8 +1,6 @@
#include "postgres.h" #include "postgres.h"
#include "access/relscan.h" #include "access/relscan.h"
#include "bitvector.h"
#include "catalog/pg_type_d.h"
#include "hnsw.h" #include "hnsw.h"
#include "pgstat.h" #include "pgstat.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"

View File

@@ -3,7 +3,6 @@
#include <math.h> #include <math.h>
#include "access/generic_xlog.h" #include "access/generic_xlog.h"
#include "catalog/pg_type_d.h"
#include "hnsw.h" #include "hnsw.h"
#include "lib/pairingheap.h" #include "lib/pairingheap.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
@@ -151,16 +150,11 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
} }
/* /*
* Get type * Get vector type
*/ */
HnswType HnswType
HnswGetType(Relation index) HnswGetType(Relation index)
{ {
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
if (typid == BITOID || typid == VARBITOID)
return HNSW_TYPE_BIT;
return HNSW_TYPE_VECTOR; return HNSW_TYPE_VECTOR;
} }

View File

@@ -2,7 +2,6 @@
#include <math.h> #include <math.h>
#include "bitvector.h"
#include "catalog/pg_type.h" #include "catalog/pg_type.h"
#include "common/shortest_dec.h" #include "common/shortest_dec.h"
#include "fmgr.h" #include "fmgr.h"
@@ -198,6 +197,8 @@ vector_in(PG_FUNCTION_ARGS)
while (pt != NULL && *stringEnd != ']') while (pt != NULL && *stringEnd != ']')
{ {
float val;
if (dim == VECTOR_MAX_DIM) if (dim == VECTOR_MAX_DIM)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
@@ -213,15 +214,23 @@ vector_in(PG_FUNCTION_ARGS)
errmsg("invalid input syntax for type vector: \"%s\"", lit))); errmsg("invalid input syntax for type vector: \"%s\"", lit)));
/* Use strtof like float4in to avoid a double-rounding problem */ /* Use strtof like float4in to avoid a double-rounding problem */
x[dim] = strtof(pt, &stringEnd); errno = 0;
CheckElement(x[dim]); val = strtof(pt, &stringEnd);
dim++;
if (stringEnd == pt) if (stringEnd == pt)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit))); errmsg("invalid input syntax for type vector: \"%s\"", lit)));
/* Check for range error like float4in */
if (errno == ERANGE && (val == 0 || isinf(val)))
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("\"%s\" is out of range for type vector", pt)));
CheckElement(val);
x[dim++] = val;
while (vector_isspace(*stringEnd)) while (vector_isspace(*stringEnd))
stringEnd++; stringEnd++;
@@ -276,9 +285,6 @@ vector_in(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
#define AppendChar(ptr, c) (*(ptr)++ = (c))
#define AppendFloat(ptr, f) ((ptr) += float_to_shortest_decimal_bufn((f), (ptr)))
/* /*
* Convert internal representation to textual representation * Convert internal representation to textual representation
*/ */
@@ -290,6 +296,7 @@ vector_out(PG_FUNCTION_ARGS)
int dim = vector->dim; int dim = vector->dim;
char *buf; char *buf;
char *ptr; char *ptr;
int n;
/* /*
* Need: * Need:
@@ -304,17 +311,21 @@ vector_out(PG_FUNCTION_ARGS)
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2); buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
ptr = buf; ptr = buf;
AppendChar(ptr, '['); *ptr = '[';
ptr++;
for (int i = 0; i < dim; i++) for (int i = 0; i < dim; i++)
{ {
if (i > 0) if (i > 0)
AppendChar(ptr, ','); {
*ptr = ',';
AppendFloat(ptr, vector->x[i]); ptr++;
} }
AppendChar(ptr, ']'); n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
ptr += n;
}
*ptr = ']';
ptr++;
*ptr = '\0'; *ptr = '\0';
PG_FREE_IF_COPY(vector, 0); PG_FREE_IF_COPY(vector, 0);
@@ -859,80 +870,6 @@ vector_mul(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
/*
* Quantize a vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(quantize_binary);
Datum
quantize_binary(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
for (int i = 0; i < a->dim; i++)
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);
}
/*
* Get a subvector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
Datum
subvector(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
int32 start = PG_GETARG_INT32(1);
int32 count = PG_GETARG_INT32(2);
int32 end = start + count;
float *ax = a->x;
Vector *result;
int dim;
/* Indexing starts at 1, like substring */
if (start < 1)
start = 1;
if (end > a->dim)
end = a->dim + 1;
dim = end - start;
CheckDim(dim);
result = InitVector(dim);
for (int i = 0; i < dim; i++)
result->x[i] = ax[start - 1 + i];
PG_RETURN_POINTER(result);
}
/*
* Concatenate vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_concat);
Datum
vector_concat(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
Vector *result;
int dim = a->dim + b->dim;
CheckDim(dim);
result = InitVector(dim);
for (int i = 0; i < a->dim; i++)
result->x[i] = a->x[i];
for (int i = 0; i < b->dim; i++)
result->x[i + a->dim] = b->x[i];
PG_RETURN_POINTER(result);
}
/* /*
* Internal helper to compare vectors * Internal helper to compare vectors
*/ */
@@ -970,6 +907,9 @@ vector_lt(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0); PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
} }
@@ -983,6 +923,9 @@ vector_le(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0); PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
} }
@@ -996,6 +939,9 @@ vector_eq(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0); PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
} }
@@ -1009,6 +955,9 @@ vector_ne(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0); PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
} }
@@ -1022,6 +971,9 @@ vector_ge(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0); PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
} }
@@ -1035,6 +987,9 @@ vector_gt(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0); PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
} }

View File

@@ -1,64 +0,0 @@
SELECT hamming_distance(B'111', B'111');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance(B'111', B'110');
hamming_distance
------------------
1
(1 row)
SELECT hamming_distance(B'111', B'100');
hamming_distance
------------------
2
(1 row)
SELECT hamming_distance(B'111', B'000');
hamming_distance
------------------
3
(1 row)
SELECT hamming_distance(B'111', B'00');
ERROR: different bit lengths 3 and 2
SELECT jaccard_distance(B'1111', B'1111');
jaccard_distance
------------------
0
(1 row)
SELECT jaccard_distance(B'1111', B'1110');
jaccard_distance
------------------
0.25
(1 row)
SELECT jaccard_distance(B'1111', B'1100');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance(B'1111', B'1000');
jaccard_distance
------------------
0.75
(1 row)
SELECT jaccard_distance(B'1111', B'0000');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance(B'1100', B'1000');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance(B'1111', B'000');
ERROR: different bit lengths 4 and 3

View File

@@ -24,38 +24,6 @@ SELECT '[1e37]'::vector * '[1e37]';
ERROR: value out of range: overflow ERROR: value out of range: overflow
SELECT '[1e-37]'::vector * '[1e-37]'; SELECT '[1e-37]'::vector * '[1e-37]';
ERROR: value out of range: underflow ERROR: value out of range: underflow
SELECT '[1,2,3]'::vector || '[4,5]'::vector;
?column?
-------------
[1,2,3,4,5]
(1 row)
SELECT array_fill(0, ARRAY[16000])::vector || '[1]'::vector;
ERROR: vector cannot have more than 16000 dimensions
SELECT '[1,2,3]'::vector < '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector < '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector = '[1,2,3]'; SELECT '[1,2,3]'::vector = '[1,2,3]';
?column? ?column?
---------- ----------
@@ -63,47 +31,7 @@ SELECT '[1,2,3]'::vector = '[1,2,3]';
(1 row) (1 row)
SELECT '[1,2,3]'::vector = '[1,2]'; SELECT '[1,2,3]'::vector = '[1,2]';
?column? ERROR: different vector dimensions 3 and 2
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector > '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector > '[1,2]';
?column?
----------
t
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2,3]'); SELECT vector_cmp('[1,2,3]', '[1,2,3]');
vector_cmp vector_cmp
------------ ------------
@@ -176,152 +104,110 @@ SELECT vector_norm('[3e37,4e37]')::real;
5e+37 5e+37
(1 row) (1 row)
SELECT l2_distance('[0,0]'::vector, '[3,4]'); SELECT l2_distance('[0,0]', '[3,4]');
l2_distance l2_distance
------------- -------------
5 5
(1 row) (1 row)
SELECT l2_distance('[0,0]'::vector, '[0,1]'); SELECT l2_distance('[0,0]', '[0,1]');
l2_distance l2_distance
------------- -------------
1 1
(1 row) (1 row)
SELECT l2_distance('[1,2]'::vector, '[3]'); SELECT l2_distance('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1 ERROR: different vector dimensions 2 and 1
SELECT l2_distance('[3e38]'::vector, '[-3e38]'); SELECT l2_distance('[3e38]', '[-3e38]');
l2_distance l2_distance
------------- -------------
Infinity Infinity
(1 row) (1 row)
SELECT inner_product('[1,2]'::vector, '[3,4]'); SELECT inner_product('[1,2]', '[3,4]');
inner_product inner_product
--------------- ---------------
11 11
(1 row) (1 row)
SELECT inner_product('[1,2]'::vector, '[3]'); SELECT inner_product('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1 ERROR: different vector dimensions 2 and 1
SELECT inner_product('[3e38]'::vector, '[3e38]'); SELECT inner_product('[3e38]', '[3e38]');
inner_product inner_product
--------------- ---------------
Infinity Infinity
(1 row) (1 row)
SELECT cosine_distance('[1,2]'::vector, '[2,4]'); SELECT cosine_distance('[1,2]', '[2,4]');
cosine_distance cosine_distance
----------------- -----------------
0 0
(1 row) (1 row)
SELECT cosine_distance('[1,2]'::vector, '[0,0]'); SELECT cosine_distance('[1,2]', '[0,0]');
cosine_distance cosine_distance
----------------- -----------------
NaN NaN
(1 row) (1 row)
SELECT cosine_distance('[1,1]'::vector, '[1,1]'); SELECT cosine_distance('[1,1]', '[1,1]');
cosine_distance cosine_distance
----------------- -----------------
0 0
(1 row) (1 row)
SELECT cosine_distance('[1,0]'::vector, '[0,2]'); SELECT cosine_distance('[1,0]', '[0,2]');
cosine_distance cosine_distance
----------------- -----------------
1 1
(1 row) (1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]'); SELECT cosine_distance('[1,1]', '[-1,-1]');
cosine_distance cosine_distance
----------------- -----------------
2 2
(1 row) (1 row)
SELECT cosine_distance('[1,2]'::vector, '[3]'); SELECT cosine_distance('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1 ERROR: different vector dimensions 2 and 1
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]'); SELECT cosine_distance('[1,1]', '[1.1,1.1]');
cosine_distance cosine_distance
----------------- -----------------
0 0
(1 row) (1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]'); SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
cosine_distance cosine_distance
----------------- -----------------
2 2
(1 row) (1 row)
SELECT cosine_distance('[3e38]'::vector, '[3e38]'); SELECT cosine_distance('[3e38]', '[3e38]');
cosine_distance cosine_distance
----------------- -----------------
NaN NaN
(1 row) (1 row)
SELECT l1_distance('[0,0]'::vector, '[3,4]'); SELECT l1_distance('[0,0]', '[3,4]');
l1_distance l1_distance
------------- -------------
7 7
(1 row) (1 row)
SELECT l1_distance('[0,0]'::vector, '[0,1]'); SELECT l1_distance('[0,0]', '[0,1]');
l1_distance l1_distance
------------- -------------
1 1
(1 row) (1 row)
SELECT l1_distance('[1,2]'::vector, '[3]'); SELECT l1_distance('[1,2]', '[3]');
ERROR: different vector dimensions 2 and 1 ERROR: different vector dimensions 2 and 1
SELECT l1_distance('[3e38]'::vector, '[-3e38]'); SELECT l1_distance('[3e38]', '[-3e38]');
l1_distance l1_distance
------------- -------------
Infinity Infinity
(1 row) (1 row)
SELECT quantize_binary('[1,0,-1]');
quantize_binary
-----------------
100
(1 row)
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]');
quantize_binary
-----------------
01001110101
(1 row)
SELECT subvector('[1,2,3,4,5]', 1, 3);
subvector
-----------
[1,2,3]
(1 row)
SELECT subvector('[1,2,3,4,5]', 3, 2);
subvector
-----------
[3,4]
(1 row)
SELECT subvector('[1,2,3,4,5]', -1, 3);
subvector
-----------
[1]
(1 row)
SELECT subvector('[1,2,3,4,5]', 3, 9);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]', 1, 0);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]', 3, -1);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]', -1, 2);
ERROR: vector must have at least 1 dimension
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v; SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
avg avg
----------- -----------

View File

@@ -1,21 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
val
-----
111
110
100
000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -1,21 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
val
------
1111
1110
1100
0000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -63,25 +63,21 @@ SELECT '[1.5e-38,-1.5e-38]'::vector;
(1 row) (1 row)
SELECT '[4e38,1]'::vector; SELECT '[4e38,1]'::vector;
ERROR: infinite value not allowed in vector ERROR: "4e38" is out of range for type vector
LINE 1: SELECT '[4e38,1]'::vector; LINE 1: SELECT '[4e38,1]'::vector;
^ ^
SELECT '[-4e38,1]'::vector; SELECT '[-4e38,1]'::vector;
ERROR: infinite value not allowed in vector ERROR: "-4e38" is out of range for type vector
LINE 1: SELECT '[-4e38,1]'::vector; LINE 1: SELECT '[-4e38,1]'::vector;
^ ^
SELECT '[1e-46,1]'::vector; SELECT '[1e-46,1]'::vector;
vector ERROR: "1e-46" is out of range for type vector
-------- LINE 1: SELECT '[1e-46,1]'::vector;
[0,1] ^
(1 row)
SELECT '[-1e-46,1]'::vector; SELECT '[-1e-46,1]'::vector;
vector ERROR: "-1e-46" is out of range for type vector
-------- LINE 1: SELECT '[-1e-46,1]'::vector;
[-0,1] ^
(1 row)
SELECT '[1,2,3'::vector; SELECT '[1,2,3'::vector;
ERROR: malformed vector literal: "[1,2,3" ERROR: malformed vector literal: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector; LINE 1: SELECT '[1,2,3'::vector;

View File

@@ -1,13 +0,0 @@
SELECT hamming_distance(B'111', B'111');
SELECT hamming_distance(B'111', B'110');
SELECT hamming_distance(B'111', B'100');
SELECT hamming_distance(B'111', B'000');
SELECT hamming_distance(B'111', B'00');
SELECT jaccard_distance(B'1111', B'1111');
SELECT jaccard_distance(B'1111', B'1110');
SELECT jaccard_distance(B'1111', B'1100');
SELECT jaccard_distance(B'1111', B'1000');
SELECT jaccard_distance(B'1111', B'0000');
SELECT jaccard_distance(B'1100', B'1000');
SELECT jaccard_distance(B'1111', B'000');

62
test/sql/functions.sql Normal file
View File

@@ -0,0 +1,62 @@
SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[3e38]'::vector + '[3e38]';
SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT '[-3e38]'::vector - '[3e38]';
SELECT '[1,2,3]'::vector * '[4,5,6]';
SELECT '[1e37]'::vector * '[1e37]';
SELECT '[1e-37]'::vector * '[1e-37]';
SELECT '[1,2,3]'::vector = '[1,2,3]';
SELECT '[1,2,3]'::vector = '[1,2]';
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
SELECT vector_cmp('[1,2]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[1,2]');
SELECT vector_cmp('[1,2]', '[2,3,4]');
SELECT vector_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT vector_norm('[3,4]');
SELECT vector_norm('[0,1]');
SELECT vector_norm('[3e37,4e37]')::real;
SELECT l2_distance('[0,0]', '[3,4]');
SELECT l2_distance('[0,0]', '[0,1]');
SELECT l2_distance('[1,2]', '[3]');
SELECT l2_distance('[3e38]', '[-3e38]');
SELECT inner_product('[1,2]', '[3,4]');
SELECT inner_product('[1,2]', '[3]');
SELECT inner_product('[3e38]', '[3e38]');
SELECT cosine_distance('[1,2]', '[2,4]');
SELECT cosine_distance('[1,2]', '[0,0]');
SELECT cosine_distance('[1,1]', '[1,1]');
SELECT cosine_distance('[1,0]', '[0,2]');
SELECT cosine_distance('[1,1]', '[-1,-1]');
SELECT cosine_distance('[1,2]', '[3]');
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
SELECT cosine_distance('[3e38]', '[3e38]');
SELECT l1_distance('[0,0]', '[3,4]');
SELECT l1_distance('[0,0]', '[0,1]');
SELECT l1_distance('[1,2]', '[3]');
SELECT l1_distance('[3e38]', '[-3e38]');
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;

View File

@@ -1,12 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
DROP TABLE t;

View File

@@ -1,12 +0,0 @@
SET enable_seqscan = off;
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
DROP TABLE t;

View File

@@ -1,86 +0,0 @@
SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[3e38]'::vector + '[3e38]';
SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT '[-3e38]'::vector - '[3e38]';
SELECT '[1,2,3]'::vector * '[4,5,6]';
SELECT '[1e37]'::vector * '[1e37]';
SELECT '[1e-37]'::vector * '[1e-37]';
SELECT '[1,2,3]'::vector || '[4,5]'::vector;
SELECT array_fill(0, ARRAY[16000])::vector || '[1]'::vector;
SELECT '[1,2,3]'::vector < '[1,2,3]';
SELECT '[1,2,3]'::vector < '[1,2]';
SELECT '[1,2,3]'::vector <= '[1,2,3]';
SELECT '[1,2,3]'::vector <= '[1,2]';
SELECT '[1,2,3]'::vector = '[1,2,3]';
SELECT '[1,2,3]'::vector = '[1,2]';
SELECT '[1,2,3]'::vector != '[1,2,3]';
SELECT '[1,2,3]'::vector != '[1,2]';
SELECT '[1,2,3]'::vector >= '[1,2,3]';
SELECT '[1,2,3]'::vector >= '[1,2]';
SELECT '[1,2,3]'::vector > '[1,2,3]';
SELECT '[1,2,3]'::vector > '[1,2]';
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
SELECT vector_cmp('[1,2]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[1,2]');
SELECT vector_cmp('[1,2]', '[2,3,4]');
SELECT vector_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT vector_norm('[3,4]');
SELECT vector_norm('[0,1]');
SELECT vector_norm('[3e37,4e37]')::real;
SELECT l2_distance('[0,0]'::vector, '[3,4]');
SELECT l2_distance('[0,0]'::vector, '[0,1]');
SELECT l2_distance('[1,2]'::vector, '[3]');
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
SELECT inner_product('[1,2]'::vector, '[3,4]');
SELECT inner_product('[1,2]'::vector, '[3]');
SELECT inner_product('[3e38]'::vector, '[3e38]');
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
SELECT cosine_distance('[1,2]'::vector, '[3]');
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
SELECT l1_distance('[0,0]'::vector, '[3,4]');
SELECT l1_distance('[0,0]'::vector, '[0,1]');
SELECT l1_distance('[1,2]'::vector, '[3]');
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
SELECT quantize_binary('[1,0,-1]');
SELECT quantize_binary('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]');
SELECT subvector('[1,2,3,4,5]', 1, 3);
SELECT subvector('[1,2,3,4,5]', 3, 2);
SELECT subvector('[1,2,3,4,5]', -1, 3);
SELECT subvector('[1,2,3,4,5]', 3, 9);
SELECT subvector('[1,2,3,4,5]', 1, 0);
SELECT subvector('[1,2,3,4,5]', 3, -1);
SELECT subvector('[1,2,3,4,5]', -1, 2);
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;

View File

@@ -1,137 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More;
my $node;
my @queries = ();
my @expected;
my $limit = 20;
my $dim = 52;
my $max = 2**$dim;
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET hnsw.ef_search = 100;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator $queries[0] LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET hnsw.ef_search = 100;
SELECT i FROM tst ORDER BY v $operator $queries[$i] LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my @expected_ids = split("\n", $expected[$i]);
my %expected_set = map { $_ => 1 } @expected_ids;
foreach (@actual_ids)
{
if (exists($expected_set{$_}))
{
$correct++;
}
}
$total += $limit;
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v bit($dim));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, (random() * $max)::bigint::bit($dim) FROM generate_series(1, 10000) i;"
);
# Generate queries
for (1 .. 20)
{
my $r = int(rand() * $max);
push(@queries, "${r}::bigint::bit($dim)");
}
# Check each index type
my @operators = ("<~>", "<\%>");
my @opclasses = ("bit_hamming_ops", "bit_jaccard_ops");
for my $i (0 .. $#operators)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Get exact results
@expected = ();
foreach (@queries)
{
# Handle ties
my $res = $node->safe_psql("postgres", qq(
WITH top AS (
SELECT v $operator $_ AS distance FROM tst ORDER BY v $operator $_ LIMIT $limit
)
SELECT i FROM tst WHERE (v $operator $_) <= (SELECT MAX(distance) FROM top)
));
push(@expected, $res);
}
# Build index serially
$node->safe_psql("postgres", qq(
SET max_parallel_maintenance_workers = 0;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
# Test approximate results
my $min = $operator eq "<\%>" ? 0.96 : 0.98;
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel in memory
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET min_parallel_table_scan_size = 1;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
# Test approximate results
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel on disk
# Set parallel_workers on table to use workers with low maintenance_work_mem
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
ALTER TABLE tst SET (parallel_workers = 2);
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '4MB';
CREATE INDEX idx ON tst USING hnsw (v $opclass);
ALTER TABLE tst RESET (parallel_workers);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
$node->safe_psql("postgres", "DROP INDEX idx;");
}
done_testing();