Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
dde3a2aacd Removed dimensions from sparsevec 2024-04-03 20:55:03 -07:00
31 changed files with 371 additions and 722 deletions

View File

@@ -1,6 +1,120 @@
name: build name: build
on: [push, pull_request] on: [push, pull_request]
jobs: jobs:
ubuntu:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 17
os: ubuntu-22.04
- postgres: 16
os: ubuntu-22.04
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-22.04
- postgres: 13
os: ubuntu-20.04
- postgres: 12
os: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- run: |
sudo apt-get update
sudo apt-get install libipc-run-perl
- run: make prove_installcheck
mac:
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 16
os: macos-14
- postgres: 14
os: macos-12
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
# Homebrew Postgres does not enable TAP tests, so need to download
- run: |
brew install cpanm
cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
env:
TAG: ${{ matrix.postgres == 16 && 'REL_16_2' || 'REL_14_11' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@15)/bin/scan-build --status-bugs make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }}
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
cd %TEMP% && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
i386:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
container:
image: debian:12
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
valgrind: valgrind:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }} if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
@@ -9,6 +123,6 @@ jobs:
- uses: ankane/setup-postgres-valgrind@v1 - uses: ankane/setup-postgres-valgrind@v1
with: with:
postgres-version: 16 postgres-version: 16
- run: make OPTFLAGS="" - run: make
- run: sudo --preserve-env=PG_CONFIG make install - run: sudo --preserve-env=PG_CONFIG make install
- run: make installcheck - run: make installcheck

111
README.md
View File

@@ -419,103 +419,6 @@ Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id); CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
``` ```
## Half Vectors
*Unreleased*
Use the `halfvec` type to store half-precision vectors
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
```
## Half Indexing
*Unreleased*
Index vectors at half precision for smaller indexes and faster build times
```sql
CREATE INDEX ON items USING hnsw ((embedding::halfvec(3)) halfvec_l2_ops);
```
Get the nearest neighbors
```sql
SELECT * FROM items ORDER BY embedding::halfvec(3) <-> '[1,2,3]' LIMIT 5;
```
## Binary Vectors
Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py))
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111');
```
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
```
Or (unreleased)
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Also supports Jaccard distance (`<%>`)
## Binary Quantization
*Unreleased*
Use expression indexing for binary quantization
```sql
CREATE INDEX ON items USING hnsw ((quantize_binary(embedding)::bit(3)) bit_hamming_ops);
```
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY quantize_binary(embedding)::bit(3) <~> quantize_binary('[1,-2,3]') LIMIT 5;
```
Re-rank by the original vectors for better recall
```sql
SELECT * FROM (
SELECT * FROM items ORDER BY quantize_binary(embedding)::bit(3) <~> quantize_binary('[1,-2,3]') LIMIT 20
) ORDER BY embedding <=> '[1,-2,3]' LIMIT 5;
```
## Sparse Vectors
*Unreleased*
Use the `sparsevec` type to store sparse vectors
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding sparsevec(5));
```
Insert vectors
```sql
INSERT INTO items (embedding) VALUES ('{1:1,3:2,5:3}/5'), ('{1:4,3:5,5:6}/5');
```
Note: The format is `{index1:value1,index2:value2,...}/dimensions` and indices start at 1 like SQL arrays
Get the nearest neighbors by L2 distance
```sql
SELECT * FROM items ORDER BY embedding <-> '{1:3,3:1,5:2}/5' LIMIT 5;
```
## Hybrid Search ## Hybrid Search
Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search. Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search.
@@ -732,6 +635,18 @@ and query with:
SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5; SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5;
``` ```
#### Are binary vectors supported?
You can store binary vectors and perform exact nearest neighbor search by Hamming distance in Postgres without an extension ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py)).
```tsql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES (B'000'), (B'111');
SELECT * FROM items ORDER BY bit_count(embedding # B'101') LIMIT 5;
```
Indexing is not currently supported.
#### Do indexes need to fit into memory? #### Do indexes need to fit into memory?
No, but like other index types, youll likely see better performance if they do. You can get the size of an index with: No, but like other index types, youll likely see better performance if they do. You can get the size of an index with:
@@ -886,7 +801,7 @@ jaccard_distance(bit, bit) → double precision | Jaccard distance | unreleased
### Sparsevec Type ### Sparsevec Type
Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each element is a single-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Sparse vectors can have up to 16,000 non-zero elements. Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each element is a single-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Sparse vectors can have up to 100,000 dimensions.
### Sparsevec Operators ### Sparsevec Operators

View File

@@ -173,9 +173,6 @@ CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -185,7 +182,6 @@ CREATE FUNCTION sparsevec_send(sparsevec) RETURNS bytea
CREATE TYPE sparsevec ( CREATE TYPE sparsevec (
INPUT = sparsevec_in, INPUT = sparsevec_in,
OUTPUT = sparsevec_out, OUTPUT = sparsevec_out,
TYPMOD_IN = sparsevec_typmod_in,
RECEIVE = sparsevec_recv, RECEIVE = sparsevec_recv,
SEND = sparsevec_send, SEND = sparsevec_send,
STORAGE = external STORAGE = external

View File

@@ -480,9 +480,6 @@ CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -492,7 +489,6 @@ CREATE FUNCTION sparsevec_send(sparsevec) RETURNS bytea
CREATE TYPE sparsevec ( CREATE TYPE sparsevec (
INPUT = sparsevec_in, INPUT = sparsevec_in,
OUTPUT = sparsevec_out, OUTPUT = sparsevec_out,
TYPMOD_IN = sparsevec_typmod_in,
RECEIVE = sparsevec_recv, RECEIVE = sparsevec_recv,
SEND = sparsevec_send, SEND = sparsevec_send,
STORAGE = external STORAGE = external

View File

@@ -22,10 +22,6 @@
#define TYPALIGN_INT 'i' #define TYPALIGN_INT 'i'
#endif #endif
#ifdef F16C_SUPPORT
#include <immintrin.h>
#endif
/* /*
* Check if half is NaN * Check if half is NaN
*/ */
@@ -103,9 +99,7 @@ pq_sendhalf(StringInfo buf, half h)
float float
HalfToFloat4(half num) HalfToFloat4(half num)
{ {
#if defined(F16C_SUPPORT) #ifdef FLT16_SUPPORT
return _cvtsh_ss(num);
#elif defined(FLT16_SUPPORT)
return (float) num; return (float) num;
#else #else
/* TODO Improve performance */ /* TODO Improve performance */
@@ -136,7 +130,7 @@ HalfToFloat4(half num)
/* Sign */ /* Sign */
result = (bin & 0x8000) << 16; result = (bin & 0x8000) << 16;
if (unlikely(exponent == 31)) if (exponent == 31)
{ {
if (mantissa == 0) if (mantissa == 0)
{ {
@@ -147,9 +141,10 @@ HalfToFloat4(half num)
{ {
/* NaN */ /* NaN */
result |= 0x7FC00000; result |= 0x7FC00000;
result |= mantissa << 13;
} }
} }
else if (unlikely(exponent == 0)) else if (exponent == 0)
{ {
/* Subnormal */ /* Subnormal */
if (mantissa != 0) if (mantissa != 0)
@@ -169,16 +164,16 @@ HalfToFloat4(half num)
} }
result |= (exponent + 127) << 23; result |= (exponent + 127) << 23;
result |= mantissa << 13;
} }
} }
else else
{ {
/* Normal */ /* Normal */
result |= (exponent - 15 + 127) << 23; result |= (exponent - 15 + 127) << 23;
result |= mantissa << 13;
} }
result |= mantissa << 13;
swapfloat.i = result; swapfloat.i = result;
return swapfloat.f; return swapfloat.f;
#endif #endif
@@ -190,9 +185,7 @@ HalfToFloat4(half num)
half half
Float4ToHalfUnchecked(float num) Float4ToHalfUnchecked(float num)
{ {
#if defined(F16C_SUPPORT) #ifdef FLT16_SUPPORT
return _cvtss_sh(num, 0);
#elif defined(FLT16_SUPPORT)
return (_Float16) num; return (_Float16) num;
#else #else
/* TODO Improve performance */ /* TODO Improve performance */
@@ -442,8 +435,6 @@ halfvec_in(PG_FUNCTION_ARGS)
while (pt != NULL && *stringEnd != ']') while (pt != NULL && *stringEnd != ']')
{ {
float val;
if (dim == HALFVEC_MAX_DIM) if (dim == HALFVEC_MAX_DIM)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
@@ -459,24 +450,15 @@ halfvec_in(PG_FUNCTION_ARGS)
errmsg("invalid input syntax for type halfvec: \"%s\"", lit))); errmsg("invalid input syntax for type halfvec: \"%s\"", lit)));
/* Use strtof like float4in to avoid a double-rounding problem */ /* Use strtof like float4in to avoid a double-rounding problem */
errno = 0; x[dim] = Float4ToHalf(strtof(pt, &stringEnd));
val = strtof(pt, &stringEnd); CheckElement(x[dim]);
dim++;
if (stringEnd == pt) if (stringEnd == pt)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type halfvec: \"%s\"", lit))); errmsg("invalid input syntax for type halfvec: \"%s\"", lit)));
x[dim] = Float4ToHalfUnchecked(val);
if ((errno == ERANGE && (isinf(val) || val == 0)) || (HalfIsInf(x[dim]) && !isinf(val)) || (HalfIsZero(x[dim]) && val != 0))
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("\"%s\" is out of range for type halfvec", pt)));
CheckElement(x[dim]);
dim++;
while (halfvec_isspace(*stringEnd)) while (halfvec_isspace(*stringEnd))
stringEnd++; stringEnd++;
@@ -801,15 +783,20 @@ vector_to_halfvec(PG_FUNCTION_ARGS)
} }
/* /*
* Get the L2 squared distance between half vectors * Get the L2 distance between half vectors
*/ */
static double PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_distance);
l2_distance_squared_internal(HalfVector * a, HalfVector * b) Datum
halfvec_l2_distance(PG_FUNCTION_ARGS)
{ {
HalfVector *a = PG_GETARG_HALFVEC_P(0);
HalfVector *b = PG_GETARG_HALFVEC_P(1);
half *ax = a->x; half *ax = a->x;
half *bx = b->x; half *bx = b->x;
float distance = 0.0; float distance = 0.0;
CheckDims(a, b);
/* Auto-vectorized */ /* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
@@ -818,22 +805,7 @@ l2_distance_squared_internal(HalfVector * a, HalfVector * b)
distance += diff * diff; distance += diff * diff;
} }
return (double) distance; PG_RETURN_FLOAT8(sqrt((double) distance));
}
/*
* Get the L2 distance between half vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_distance);
Datum
halfvec_l2_distance(PG_FUNCTION_ARGS)
{
HalfVector *a = PG_GETARG_HALFVEC_P(0);
HalfVector *b = PG_GETARG_HALFVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(sqrt(l2_distance_squared_internal(a, b)));
} }
/* /*
@@ -845,51 +817,21 @@ halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
{ {
HalfVector *a = PG_GETARG_HALFVEC_P(0); HalfVector *a = PG_GETARG_HALFVEC_P(0);
HalfVector *b = PG_GETARG_HALFVEC_P(1); HalfVector *b = PG_GETARG_HALFVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(l2_distance_squared_internal(a, b));
}
/*
* Get the inner product of two half vectors
*/
static double
inner_product_internal(HalfVector * a, HalfVector * b)
{
half *ax = a->x; half *ax = a->x;
half *bx = b->x; half *bx = b->x;
float distance = 0.0; float distance = 0.0;
#if defined(F16C_SUPPORT) && defined(__FMA__) CheckDims(a, b);
int i;
float s[8];
int count = (a->dim / 8) * 8;
__m256 dist = _mm256_setzero_ps();
for (i = 0; i < count; i += 8)
{
__m128i axi = _mm_loadu_si128((__m128i *) (ax + i));
__m128i bxi = _mm_loadu_si128((__m128i *) (bx + i));
__m256 axs = _mm256_cvtph_ps(axi);
__m256 bxs = _mm256_cvtph_ps(bxi);
dist = _mm256_fmadd_ps(axs, bxs, dist);
}
_mm256_store_ps(s, dist);
distance = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
for (; i < a->dim; i++)
distance += HalfToFloat4(ax[i]) * HalfToFloat4(bx[i]);
#else
/* Auto-vectorized */ /* Auto-vectorized */
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
distance += HalfToFloat4(ax[i]) * HalfToFloat4(bx[i]); {
#endif float diff = HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]);
return (double) distance; distance += diff * diff;
}
PG_RETURN_FLOAT8((double) distance);
} }
/* /*
@@ -901,10 +843,17 @@ halfvec_inner_product(PG_FUNCTION_ARGS)
{ {
HalfVector *a = PG_GETARG_HALFVEC_P(0); HalfVector *a = PG_GETARG_HALFVEC_P(0);
HalfVector *b = PG_GETARG_HALFVEC_P(1); HalfVector *b = PG_GETARG_HALFVEC_P(1);
half *ax = a->x;
half *bx = b->x;
float distance = 0.0;
CheckDims(a, b); CheckDims(a, b);
PG_RETURN_FLOAT8(inner_product_internal(a, b)); /* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += HalfToFloat4(ax[i]) * HalfToFloat4(bx[i]);
PG_RETURN_FLOAT8((double) distance);
} }
/* /*
@@ -916,10 +865,17 @@ halfvec_negative_inner_product(PG_FUNCTION_ARGS)
{ {
HalfVector *a = PG_GETARG_HALFVEC_P(0); HalfVector *a = PG_GETARG_HALFVEC_P(0);
HalfVector *b = PG_GETARG_HALFVEC_P(1); HalfVector *b = PG_GETARG_HALFVEC_P(1);
half *ax = a->x;
half *bx = b->x;
float distance = 0.0;
CheckDims(a, b); CheckDims(a, b);
PG_RETURN_FLOAT8(-inner_product_internal(a, b)); /* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += HalfToFloat4(ax[i]) * HalfToFloat4(bx[i]);
PG_RETURN_FLOAT8((double) distance * -1);
} }
/* /*

View File

@@ -7,9 +7,7 @@
#include "vector.h" #include "vector.h"
#if defined(__F16C__) #ifdef __FLT16_MAX__
#define F16C_SUPPORT
#elif defined(__FLT16_MAX__)
#define FLT16_SUPPORT #define FLT16_SUPPORT
#endif #endif

View File

@@ -681,8 +681,6 @@ GetMaxDimensions(HnswType type)
maxDimensions *= 2; maxDimensions *= 2;
else if (type == HNSW_TYPE_BIT) else if (type == HNSW_TYPE_BIT)
maxDimensions *= 32; maxDimensions *= 32;
else if (type == HNSW_TYPE_SPARSEVEC)
maxDimensions = INT_MAX;
return maxDimensions; return maxDimensions;
} }
@@ -693,8 +691,6 @@ GetMaxDimensions(HnswType type)
static void static void
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum) InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
{ {
int maxDimensions;
buildstate->heap = heap; buildstate->heap = heap;
buildstate->index = index; buildstate->index = index;
buildstate->indexInfo = indexInfo; buildstate->indexInfo = indexInfo;
@@ -705,14 +701,17 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
buildstate->efConstruction = HnswGetEfConstruction(index); buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod; buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
maxDimensions = GetMaxDimensions(buildstate->type);
/* Require column to have dimensions to be indexed */ if (buildstate->type != HNSW_TYPE_SPARSEVEC)
if (buildstate->dimensions < 0) {
elog(ERROR, "column does not have dimensions"); int maxDimensions = GetMaxDimensions(buildstate->type);
if (buildstate->dimensions > maxDimensions) if (buildstate->dimensions < 0)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", maxDimensions); elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > maxDimensions)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", maxDimensions);
}
if (buildstate->efConstruction < 2 * buildstate->m) if (buildstate->efConstruction < 2 * buildstate->m)
elog(ERROR, "ef_construction must be greater than or equal to 2 * m"); elog(ERROR, "ef_construction must be greater than or equal to 2 * m");

View File

@@ -158,10 +158,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock); UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
so->first = false; so->first = false;
#if defined(HNSW_MEMORY) && PG_VERSION_NUM >= 130000
elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(so->tmpCtx, false) / (1024 * 1024));
#endif
} }
while (list_length(so->w) > 0) while (list_length(so->w) > 0)

View File

@@ -163,9 +163,9 @@ HnswGetType(Relation index)
Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid; Oid typid = TupleDescAttr(index->rd_att, 0)->atttypid;
HeapTuple tuple; HeapTuple tuple;
Form_pg_type type; Form_pg_type type;
HnswType result; int result;
if (typid == BITOID) if (typid == BITOID || typid == VARBITOID)
return HNSW_TYPE_BIT; return HNSW_TYPE_BIT;
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid)); tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid));
@@ -180,10 +180,7 @@ HnswGetType(Relation index)
else if (strcmp(NameStr(type->typname), "sparsevec") == 0) else if (strcmp(NameStr(type->typname), "sparsevec") == 0)
result = HNSW_TYPE_SPARSEVEC; result = HNSW_TYPE_SPARSEVEC;
else else
{ elog(ERROR, "Unsupported type");
ReleaseSysCache(tuple);
elog(ERROR, "type not supported for hnsw index");
}
ReleaseSysCache(tuple); ReleaseSysCache(tuple);
@@ -232,7 +229,7 @@ HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type)
else if (type == HNSW_TYPE_SPARSEVEC) else if (type == HNSW_TYPE_SPARSEVEC)
{ {
SparseVector *v = DatumGetSparseVector(*value); SparseVector *v = DatumGetSparseVector(*value);
SparseVector *result = InitSparseVector(v->dim, v->nnz); SparseVector *result = InitSparseVector(v->nnz);
float *vx = SPARSEVEC_VALUES(v); float *vx = SPARSEVEC_VALUES(v);
float *rx = SPARSEVEC_VALUES(result); float *rx = SPARSEVEC_VALUES(result);

View File

@@ -18,86 +18,35 @@
#include "utils/builtins.h" #include "utils/builtins.h"
#endif #endif
/*
* Ensure same dimensions
*/
static inline void
CheckDims(SparseVector * a, SparseVector * b)
{
if (a->dim != b->dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("different sparsevec dimensions %d and %d", a->dim, b->dim)));
}
/*
* Ensure expected dimensions
*/
static inline void
CheckExpectedDim(int32 typmod, int dim)
{
if (typmod != -1 && typmod != dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("expected %d dimensions, not %d", typmod, dim)));
}
/*
* Ensure valid dimensions
*/
static inline void
CheckDim(int dim)
{
if (dim < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec must have at least 1 dimension")));
if (dim > SPARSEVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more than %d dimensions", SPARSEVEC_MAX_DIM)));
}
/* /*
* Ensure valid nnz * Ensure valid nnz
*/ */
static inline void static inline void
CheckNnz(int nnz, int dim) CheckNnz(int nnz)
{ {
if (nnz < 0) if (nnz < 0)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION), (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("sparsevec cannot have negative number of elements"))); errmsg("sparsevec must have at least one element")));
if (nnz > SPARSEVEC_MAX_NNZ) if (nnz > SPARSEVEC_MAX_NNZ)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more than %d non-zero elements", SPARSEVEC_MAX_NNZ))); errmsg("sparsevec cannot have more elements than non-zero elements")));
if (nnz > dim)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("sparsevec cannot have more elements than dimensions")));
} }
/* /*
* Ensure valid index * Ensure valid index
*/ */
static inline void static inline void
CheckIndex(int32 *indices, int i, int dim) CheckIndex(int32 *indices, int i)
{ {
int32 index = indices[i]; int32 index = indices[i];
if (index < 1) if (index < 0)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION), (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("index must be greater than zero"))); errmsg("index must not be negative")));
if (index > dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("index must be less than or equal to dimensions")));
if (i > 0) if (i > 0)
{ {
@@ -134,7 +83,7 @@ CheckElement(float value)
* Allocate and initialize a new sparse vector * Allocate and initialize a new sparse vector
*/ */
SparseVector * SparseVector *
InitSparseVector(int dim, int nnz) InitSparseVector(int nnz)
{ {
SparseVector *result; SparseVector *result;
int size; int size;
@@ -142,7 +91,6 @@ InitSparseVector(int dim, int nnz)
size = SPARSEVEC_SIZE(nnz); size = SPARSEVEC_SIZE(nnz);
result = (SparseVector *) palloc0(size); result = (SparseVector *) palloc0(size);
SET_VARSIZE(result, size); SET_VARSIZE(result, size);
result->dim = dim;
result->nnz = nnz; result->nnz = nnz;
return result; return result;
@@ -172,8 +120,6 @@ Datum
sparsevec_in(PG_FUNCTION_ARGS) sparsevec_in(PG_FUNCTION_ARGS)
{ {
char *lit = PG_GETARG_CSTRING(0); char *lit = PG_GETARG_CSTRING(0);
int32 typmod = PG_GETARG_INT32(2);
int dim;
char *pt; char *pt;
char *stringEnd; char *stringEnd;
SparseVector *result; SparseVector *result;
@@ -245,7 +191,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit))); errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
if (errno == ERANGE || index < 1 || index > INT_MAX) if (errno == ERANGE || index < 0 || index > INT_MAX)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("index \"%ld\" is out of range for type sparsevec", index))); errmsg("index \"%ld\" is out of range for type sparsevec", index)));
@@ -307,24 +253,6 @@ sparsevec_in(PG_FUNCTION_ARGS)
stringEnd++; stringEnd++;
if (*stringEnd != '/')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed sparsevec literal: \"%s\"", lit),
errdetail("Unexpected end of input.")));
stringEnd++;
/* Use similar logic as int2vectorin */
errno = 0;
pt = stringEnd;
dim = strtol(pt, &stringEnd, 10);
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type sparsevec: \"%s\"", lit)));
/* Only whitespace is allowed after the closing brace */ /* Only whitespace is allowed after the closing brace */
while (sparsevec_isspace(*stringEnd)) while (sparsevec_isspace(*stringEnd))
stringEnd++; stringEnd++;
@@ -333,21 +261,18 @@ sparsevec_in(PG_FUNCTION_ARGS)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed sparsevec literal: \"%s\"", lit), errmsg("malformed sparsevec literal: \"%s\"", lit),
errdetail("Junk after closing."))); errdetail("Junk after closing right brace.")));
pfree(litcopy); pfree(litcopy);
CheckDim(dim); result = InitSparseVector(nnz);
CheckExpectedDim(typmod, dim);
result = InitSparseVector(dim, nnz);
rvalues = SPARSEVEC_VALUES(result); rvalues = SPARSEVEC_VALUES(result);
for (int i = 0; i < nnz; i++) for (int i = 0; i < nnz; i++)
{ {
result->indices[i] = indices[i]; result->indices[i] = indices[i];
rvalues[i] = values[i]; rvalues[i] = values[i];
CheckIndex(result->indices, i, dim); CheckIndex(result->indices, i);
CheckElement(rvalues[i]); CheckElement(rvalues[i]);
} }
@@ -392,11 +317,9 @@ sparsevec_out(PG_FUNCTION_ARGS)
* *
* nnz - 1 bytes for , * nnz - 1 bytes for ,
* *
* 10 bytes for dimensions * 3 bytes for {, }, and \0
*
* 4 bytes for {, }, /, and \0
*/ */
buf = (char *) palloc((11 + FLOAT_SHORTEST_DECIMAL_LEN) * sparsevec->nnz + 13); buf = (char *) palloc((11 + FLOAT_SHORTEST_DECIMAL_LEN) * sparsevec->nnz + 2);
ptr = buf; ptr = buf;
AppendChar(ptr, '{'); AppendChar(ptr, '{');
@@ -412,45 +335,12 @@ sparsevec_out(PG_FUNCTION_ARGS)
} }
AppendChar(ptr, '}'); AppendChar(ptr, '}');
AppendChar(ptr, '/');
AppendInt(ptr, sparsevec->dim);
*ptr = '\0'; *ptr = '\0';
PG_FREE_IF_COPY(sparsevec, 0); PG_FREE_IF_COPY(sparsevec, 0);
PG_RETURN_CSTRING(buf); PG_RETURN_CSTRING(buf);
} }
/*
* Convert type modifier
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
Datum
sparsevec_typmod_in(PG_FUNCTION_ARGS)
{
ArrayType *ta = PG_GETARG_ARRAYTYPE_P(0);
int32 *tl;
int n;
tl = ArrayGetIntegerTypmods(ta, &n);
if (n != 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("invalid type modifier")));
if (*tl < 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions for type sparsevec must be at least 1")));
if (*tl > SPARSEVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions for type sparsevec cannot exceed %d", SPARSEVEC_MAX_DIM)));
PG_RETURN_INT32(*tl);
}
/* /*
* Convert external binary representation to internal representation * Convert external binary representation to internal representation
*/ */
@@ -459,33 +349,30 @@ Datum
sparsevec_recv(PG_FUNCTION_ARGS) sparsevec_recv(PG_FUNCTION_ARGS)
{ {
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
int32 typmod = PG_GETARG_INT32(2);
SparseVector *result; SparseVector *result;
int32 dim;
int32 nnz; int32 nnz;
int32 unused; int32 unused;
int32 unused2;
float *values; float *values;
dim = pq_getmsgint(buf, sizeof(int32));
nnz = pq_getmsgint(buf, sizeof(int32)); nnz = pq_getmsgint(buf, sizeof(int32));
unused = pq_getmsgint(buf, sizeof(int32)); unused = pq_getmsgint(buf, sizeof(int32));
unused2 = pq_getmsgint(buf, sizeof(int32));
CheckDim(dim); CheckNnz(nnz);
CheckNnz(nnz, dim);
CheckExpectedDim(typmod, dim);
if (unused != 0) if (unused != 0 || unused2 != 0)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION), (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("expected unused to be 0, not %d", unused))); errmsg("expected unused to be 0, not %d", unused)));
result = InitSparseVector(dim, nnz); result = InitSparseVector(nnz);
values = SPARSEVEC_VALUES(result); values = SPARSEVEC_VALUES(result);
for (int i = 0; i < nnz; i++) for (int i = 0; i < nnz; i++)
{ {
result->indices[i] = pq_getmsgint(buf, sizeof(int32)); result->indices[i] = pq_getmsgint(buf, sizeof(int32));
CheckIndex(result->indices, i, dim); CheckIndex(result->indices, i);
} }
for (int i = 0; i < nnz; i++) for (int i = 0; i < nnz; i++)
@@ -509,9 +396,9 @@ sparsevec_send(PG_FUNCTION_ARGS)
StringInfoData buf; StringInfoData buf;
pq_begintypsend(&buf); pq_begintypsend(&buf);
pq_sendint(&buf, svec->dim, sizeof(int32));
pq_sendint(&buf, svec->nnz, sizeof(int32)); pq_sendint(&buf, svec->nnz, sizeof(int32));
pq_sendint(&buf, svec->unused, sizeof(int32)); pq_sendint(&buf, svec->unused, sizeof(int32));
pq_sendint(&buf, svec->unused2, sizeof(int32));
for (int i = 0; i < svec->nnz; i++) for (int i = 0; i < svec->nnz; i++)
pq_sendint(&buf, svec->indices[i], sizeof(int32)); pq_sendint(&buf, svec->indices[i], sizeof(int32));
for (int i = 0; i < svec->nnz; i++) for (int i = 0; i < svec->nnz; i++)
@@ -522,16 +409,12 @@ sparsevec_send(PG_FUNCTION_ARGS)
/* /*
* Convert sparse vector to sparse vector * Convert sparse vector to sparse vector
* This is needed to check the type modifier
*/ */
PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec);
Datum Datum
sparsevec(PG_FUNCTION_ARGS) sparsevec(PG_FUNCTION_ARGS)
{ {
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0); SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
CheckExpectedDim(typmod, svec->dim);
PG_RETURN_POINTER(svec); PG_RETURN_POINTER(svec);
} }
@@ -544,23 +427,20 @@ Datum
vector_to_sparsevec(PG_FUNCTION_ARGS) vector_to_sparsevec(PG_FUNCTION_ARGS)
{ {
Vector *vec = PG_GETARG_VECTOR_P(0); Vector *vec = PG_GETARG_VECTOR_P(0);
int32 typmod = PG_GETARG_INT32(1);
SparseVector *result; SparseVector *result;
int dim = vec->dim; int dim = vec->dim;
int nnz = 0; int nnz = 0;
float *values; float *values;
int j = 0; int j = 0;
CheckDim(dim);
CheckExpectedDim(typmod, dim);
for (int i = 0; i < dim; i++) for (int i = 0; i < dim; i++)
{ {
if (vec->x[i] != 0) if (vec->x[i] != 0)
nnz++; nnz++;
} }
result = InitSparseVector(dim, nnz); CheckNnz(nnz);
result = InitSparseVector(nnz);
values = SPARSEVEC_VALUES(result); values = SPARSEVEC_VALUES(result);
for (int i = 0; i < dim; i++) for (int i = 0; i < dim; i++)
{ {
@@ -570,7 +450,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
if (j == nnz) if (j == nnz)
elog(ERROR, "safety check failed"); elog(ERROR, "safety check failed");
result->indices[j] = i + 1; result->indices[j] = i;
values[j] = vec->x[i]; values[j] = vec->x[i];
j++; j++;
} }
@@ -580,7 +460,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
} }
/* /*
* Get the L2 squared distance between sparse vectors * Get the L2 squared distance between sparse vectors
*/ */
static double static double
l2_distance_squared_internal(SparseVector * a, SparseVector * b) l2_distance_squared_internal(SparseVector * a, SparseVector * b)
@@ -637,8 +517,6 @@ sparsevec_l2_distance(PG_FUNCTION_ARGS)
SparseVector *a = PG_GETARG_SPARSEVEC_P(0); SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1); SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(sqrt(l2_distance_squared_internal(a, b))); PG_RETURN_FLOAT8(sqrt(l2_distance_squared_internal(a, b)));
} }
@@ -653,8 +531,6 @@ sparsevec_l2_squared_distance(PG_FUNCTION_ARGS)
SparseVector *a = PG_GETARG_SPARSEVEC_P(0); SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1); SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(l2_distance_squared_internal(a, b)); PG_RETURN_FLOAT8(l2_distance_squared_internal(a, b));
} }
@@ -704,8 +580,6 @@ sparsevec_inner_product(PG_FUNCTION_ARGS)
SparseVector *a = PG_GETARG_SPARSEVEC_P(0); SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1); SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(inner_product_internal(a, b)); PG_RETURN_FLOAT8(inner_product_internal(a, b));
} }
@@ -719,8 +593,6 @@ sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
SparseVector *a = PG_GETARG_SPARSEVEC_P(0); SparseVector *a = PG_GETARG_SPARSEVEC_P(0);
SparseVector *b = PG_GETARG_SPARSEVEC_P(1); SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(-inner_product_internal(a, b)); PG_RETURN_FLOAT8(-inner_product_internal(a, b));
} }
@@ -739,8 +611,6 @@ sparsevec_cosine_distance(PG_FUNCTION_ARGS)
float normb = 0.0; float normb = 0.0;
double similarity; double similarity;
CheckDims(a, b);
similarity = inner_product_internal(a, b); similarity = inner_product_internal(a, b);
/* Auto-vectorized */ /* Auto-vectorized */

View File

@@ -1,8 +1,7 @@
#ifndef SPARSEVEC_H #ifndef SPARSEVEC_H
#define SPARSEVEC_H #define SPARSEVEC_H
#define SPARSEVEC_MAX_DIM 100000 #define SPARSEVEC_MAX_NNZ 100000
#define SPARSEVEC_MAX_NNZ 16000
/* Ensure values are aligned */ /* Ensure values are aligned */
#define SPARSEVEC_SIZE(_nnz) (offsetof(SparseVector, indices) + MAXALIGN((_nnz) * sizeof(int32)) + (_nnz * sizeof(float))) #define SPARSEVEC_SIZE(_nnz) (offsetof(SparseVector, indices) + MAXALIGN((_nnz) * sizeof(int32)) + (_nnz * sizeof(float)))
@@ -14,12 +13,12 @@
typedef struct SparseVector typedef struct SparseVector
{ {
int32 vl_len_; /* varlena header (do not touch directly!) */ int32 vl_len_; /* varlena header (do not touch directly!) */
int32 dim; /* number of dimensions */
int32 nnz; int32 nnz;
int32 unused; int32 unused;
int32 unused2;
int32 indices[FLEXIBLE_ARRAY_MEMBER]; int32 indices[FLEXIBLE_ARRAY_MEMBER];
} SparseVector; } SparseVector;
SparseVector *InitSparseVector(int dim, int nnz); SparseVector *InitSparseVector(int nnz);
#endif #endif

View File

@@ -1236,15 +1236,23 @@ sparsevec_to_vector(PG_FUNCTION_ARGS)
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0); SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
int32 typmod = PG_GETARG_INT32(1); int32 typmod = PG_GETARG_INT32(1);
Vector *result; Vector *result;
int dim = svec->dim; int dim;
float *values = SPARSEVEC_VALUES(svec); float *values = SPARSEVEC_VALUES(svec);
int maxIndex = svec->nnz == 0 ? -1 : svec->indices[svec->nnz - 1];
if (typmod == -1)
dim = maxIndex + 1;
else
dim = typmod;
CheckDim(dim); CheckDim(dim);
CheckExpectedDim(typmod, dim);
if (dim < maxIndex + 1)
elog(ERROR, "Vector must have at least %d dimensions", maxIndex + 1);
result = InitVector(dim); result = InitVector(dim);
for (int i = 0; i < svec->nnz; i++) for (int i = 0; i < svec->nnz; i++)
result->x[svec->indices[i] - 1] = values[i]; result->x[svec->indices[i]] = values[i];
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }

View File

@@ -1,104 +1,64 @@
SELECT hamming_distance('111', '111'); SELECT hamming_distance(B'111', B'111');
hamming_distance hamming_distance
------------------ ------------------
0 0
(1 row) (1 row)
SELECT hamming_distance('111', '110'); SELECT hamming_distance(B'111', B'110');
hamming_distance hamming_distance
------------------ ------------------
1 1
(1 row) (1 row)
SELECT hamming_distance('111', '100'); SELECT hamming_distance(B'111', B'100');
hamming_distance hamming_distance
------------------ ------------------
2 2
(1 row) (1 row)
SELECT hamming_distance('111', '000'); SELECT hamming_distance(B'111', B'000');
hamming_distance hamming_distance
------------------ ------------------
3 3
(1 row) (1 row)
SELECT hamming_distance('10101010101010101010', '01010101010101010101'); SELECT hamming_distance(B'111', B'00');
hamming_distance
------------------
20
(1 row)
SELECT hamming_distance('', '');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance('111', '00');
ERROR: different bit lengths 3 and 2 ERROR: different bit lengths 3 and 2
SELECT hamming_distance('111', '000'::varbit(4)); SELECT jaccard_distance(B'1111', B'1111');
hamming_distance
------------------
3
(1 row)
SELECT hamming_distance('111', '0000'::varbit(4));
ERROR: different bit lengths 3 and 4
SELECT jaccard_distance('1111', '1111');
jaccard_distance jaccard_distance
------------------ ------------------
0 0
(1 row) (1 row)
SELECT jaccard_distance('1111', '1110'); SELECT jaccard_distance(B'1111', B'1110');
jaccard_distance jaccard_distance
------------------ ------------------
0.25 0.25
(1 row) (1 row)
SELECT jaccard_distance('1111', '1100'); SELECT jaccard_distance(B'1111', B'1100');
jaccard_distance jaccard_distance
------------------ ------------------
0.5 0.5
(1 row) (1 row)
SELECT jaccard_distance('1111', '1000'); SELECT jaccard_distance(B'1111', B'1000');
jaccard_distance jaccard_distance
------------------ ------------------
0.75 0.75
(1 row) (1 row)
SELECT jaccard_distance('1111', '0000'); SELECT jaccard_distance(B'1111', B'0000');
jaccard_distance jaccard_distance
------------------ ------------------
1 1
(1 row) (1 row)
SELECT jaccard_distance('1100', '1000'); SELECT jaccard_distance(B'1100', B'1000');
jaccard_distance jaccard_distance
------------------ ------------------
0.5 0.5
(1 row) (1 row)
SELECT jaccard_distance('10101010101010101010', '01010101010101010101'); SELECT jaccard_distance(B'1111', B'000');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('', '');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1111', '000');
ERROR: different bit lengths 4 and 3 ERROR: different bit lengths 4 and 3
SELECT jaccard_distance('1111', '0000'::varbit(5));
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1111', '00000'::varbit(5));
ERROR: different bit lengths 4 and 5

View File

@@ -51,21 +51,17 @@ SELECT '[65519,-65519]'::halfvec;
(1 row) (1 row)
SELECT '[65520,-65520]'::halfvec; SELECT '[65520,-65520]'::halfvec;
ERROR: "65520" is out of range for type halfvec ERROR: value out of range: overflow
LINE 1: SELECT '[65520,-65520]'::halfvec; LINE 1: SELECT '[65520,-65520]'::halfvec;
^ ^
SELECT '[1e-8,-1e-8]'::halfvec; SELECT '[1e-8,-1e-8]'::halfvec;
ERROR: "1e-8" is out of range for type halfvec ERROR: value out of range: underflow
LINE 1: SELECT '[1e-8,-1e-8]'::halfvec; LINE 1: SELECT '[1e-8,-1e-8]'::halfvec;
^ ^
SELECT '[4e38,1]'::halfvec; SELECT '[4e38,1]'::halfvec;
ERROR: "4e38" is out of range for type halfvec ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[4e38,1]'::halfvec; LINE 1: SELECT '[4e38,1]'::halfvec;
^ ^
SELECT '[1e-46,1]'::halfvec;
ERROR: "1e-46" is out of range for type halfvec
LINE 1: SELECT '[1e-46,1]'::halfvec;
^
SELECT '[1,2,3'::halfvec; SELECT '[1,2,3'::halfvec;
ERROR: malformed halfvec literal: "[1,2,3" ERROR: malformed halfvec literal: "[1,2,3"
LINE 1: SELECT '[1,2,3'::halfvec; LINE 1: SELECT '[1,2,3'::halfvec;

View File

@@ -19,9 +19,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
(1 row) (1 row)
DROP TABLE t; DROP TABLE t;
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: type not supported for hnsw index
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
DROP TABLE t;

View File

@@ -1,17 +1,17 @@
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL); INSERT INTO t (val) VALUES ('{}'), ('{0:1,1:2,2:3}'), ('{0:1,1:1,2:1}'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops); CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3'); INSERT INTO t (val) VALUES ('{0:1,1:2,2:4}');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <=> '{0:3,1:3,2:3}';
val val
----------------- ---------------
{1:1,2:1,3:1}/3 {0:1,1:1,2:1}
{1:1,2:2,3:3}/3 {0:1,1:2,2:3}
{1:1,2:2,3:4}/3 {0:1,1:2,2:4}
(3 rows) (3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}') t2;
count count
------- -------
3 3

View File

@@ -1,15 +1,15 @@
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL); INSERT INTO t (val) VALUES ('{}'), ('{0:1,1:2,2:3}'), ('{0:1,1:1,2:1}'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops); CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3'); INSERT INTO t (val) VALUES ('{0:1,1:2,2:4}');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <#> '{0:3,1:3,2:3}';
val val
----------------- ---------------
{1:1,2:2,3:4}/3 {0:1,1:2,2:4}
{1:1,2:2,3:3}/3 {0:1,1:2,2:3}
{1:1,2:1,3:1}/3 {0:1,1:1,2:1}
{}/3 {}
(4 rows) (4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;

View File

@@ -1,15 +1,15 @@
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL); INSERT INTO t (val) VALUES ('{}'), ('{0:1,1:2,2:3}'), ('{0:1,1:1,2:1}'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops); CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3'); INSERT INTO t (val) VALUES ('{0:1,1:2,2:4}');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <-> '{0:3,1:3,2:3}';
val val
----------------- ---------------
{1:1,2:2,3:3}/3 {0:1,1:2,2:3}
{1:1,2:2,3:4}/3 {0:1,1:2,2:4}
{1:1,2:1,3:1}/3 {0:1,1:1,2:1}
{}/3 {}
(4 rows) (4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
@@ -25,14 +25,14 @@ SELECT COUNT(*) FROM t;
(1 row) (1 row)
TRUNCATE t; TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <-> '{0:3,1:3,2:3}';
val val
----- -----
(0 rows) (0 rows)
DROP TABLE t; DROP TABLE t;
-- TODO move -- TODO move
CREATE TABLE t (val sparsevec(1001)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec); INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops); CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index

View File

@@ -1,62 +1,60 @@
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2'); SELECT l2_distance('{}'::sparsevec, '{0:3,1:4}');
l2_distance l2_distance
------------- -------------
5 5
(1 row) (1 row)
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2'); SELECT l2_distance('{}'::sparsevec, '{1:1}');
l2_distance l2_distance
------------- -------------
1 1
(1 row) (1 row)
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2'; SELECT '{}'::sparsevec <-> '{0:3,1:4}';
?column? ?column?
---------- ----------
5 5
(1 row) (1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2'); SELECT inner_product('{0:1,1:2}'::sparsevec, '{0:2,1:4}');
inner_product inner_product
--------------- ---------------
10 10
(1 row) (1 row)
SELECT sparsevec_negative_inner_product('{1:1,2:2}/2', '{1:2,2:4}/2'); SELECT sparsevec_negative_inner_product('{0:1,1:2}', '{0:2,1:4}');
sparsevec_negative_inner_product sparsevec_negative_inner_product
---------------------------------- ----------------------------------
-10 -10
(1 row) (1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2'); SELECT cosine_distance('{0:1,1:2}'::sparsevec, '{0:2,1:4}');
cosine_distance cosine_distance
----------------- -----------------
0 0
(1 row) (1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2'); SELECT cosine_distance('{0:1,1:2}'::sparsevec, '{}');
cosine_distance cosine_distance
----------------- -----------------
NaN NaN
(1 row) (1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2'); SELECT cosine_distance('{0:1,1:1}'::sparsevec, '{0:-1,1:-1}');
cosine_distance cosine_distance
----------------- -----------------
2 2
(1 row) (1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2'); SELECT cosine_distance('{0:1}'::sparsevec, '{1:2}');
cosine_distance cosine_distance
----------------- -----------------
1 1
(1 row) (1 row)
SELECT cosine_distance('{}/1'::sparsevec, '{}/1'); SELECT cosine_distance('{}'::sparsevec, '{}');
cosine_distance cosine_distance
----------------- -----------------
NaN NaN
(1 row) (1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{1:1}/3');
ERROR: different sparsevec dimensions 2 and 3

View File

@@ -1,62 +1,64 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec; SELECT '{0:1.5,2:3.5}'::sparsevec;
sparsevec sparsevec
----------------- ---------------
{1:1.5,3:3.5}/5 {0:1.5,2:3.5}
(1 row) (1 row)
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector; SELECT '{0:1.5,2:3.5}'::sparsevec::vector;
vector
-------------
[1.5,0,3.5]
(1 row)
SELECT '{0:1.5,2:3.5}'::sparsevec::vector(5);
vector vector
----------------- -----------------
[1.5,0,3.5,0,0] [1.5,0,3.5,0,0]
(1 row) (1 row)
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(5); SELECT '{0:1.5,2:3.5}'::sparsevec::vector(4);
vector vector
----------------- ---------------
[1.5,0,3.5,0,0] [1.5,0,3.5,0]
(1 row) (1 row)
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(4); SELECT '{0:1.5,2:3.5}'::sparsevec::vector(2);
ERROR: expected 4 dimensions, not 5 ERROR: Vector must have at least 3 dimensions
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec; SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
sparsevec sparsevec
----------------- ---------------
{2:1.5,4:3.5}/5 {1:1.5,3:3.5}
(1 row) (1 row)
SELECT '{1:0,2:1,3:0}/3'::sparsevec; SELECT '{0:0,1:1,2:0}'::sparsevec;
sparsevec sparsevec
----------- -----------
{2:1}/3 {1:1}
(1 row) (1 row)
SELECT '{2:1,1:1}/2'::sparsevec; SELECT '{1:1,0:1}'::sparsevec;
ERROR: indexes must be in ascending order ERROR: indexes must be in ascending order
LINE 1: SELECT '{2:1,1:1}/2'::sparsevec; LINE 1: SELECT '{1:1,0:1}'::sparsevec;
^ ^
SELECT '{}/5'::sparsevec; SELECT '{}'::sparsevec;
sparsevec sparsevec
----------- -----------
{}/5 {}
(1 row) (1 row)
SELECT '{}/-1'::sparsevec; SELECT '{}'::sparsevec::vector;
ERROR: sparsevec must have at least 1 dimension ERROR: vector must have at least 1 dimension
LINE 1: SELECT '{}/-1'::sparsevec; SELECT '{-1:1}'::sparsevec;
ERROR: index "-1" is out of range for type sparsevec
LINE 1: SELECT '{-1:1}'::sparsevec;
^ ^
SELECT '{}/100001'::sparsevec; SELECT '{1:1}'::sparsevec;
ERROR: sparsevec cannot have more than 100000 dimensions sparsevec
LINE 1: SELECT '{}/100001'::sparsevec; -----------
^ {1:1}
SELECT '{}/16001'::sparsevec::vector; (1 row)
ERROR: vector cannot have more than 16000 dimensions
SELECT '{0:1}/1'::sparsevec; SELECT '{}'::sparsevec(2);
ERROR: index "0" is out of range for type sparsevec ERROR: type modifier is not allowed for type "sparsevec"
LINE 1: SELECT '{0:1}/1'::sparsevec; LINE 1: SELECT '{}'::sparsevec(2);
^ ^
SELECT '{2:1}/1'::sparsevec;
ERROR: index must be less than or equal to dimensions
LINE 1: SELECT '{2:1}/1'::sparsevec;
^
SELECT '{}/1'::sparsevec(2);
ERROR: expected 2 dimensions, not 1

View File

@@ -1,21 +1,13 @@
SELECT hamming_distance('111', '111'); SELECT hamming_distance(B'111', B'111');
SELECT hamming_distance('111', '110'); SELECT hamming_distance(B'111', B'110');
SELECT hamming_distance('111', '100'); SELECT hamming_distance(B'111', B'100');
SELECT hamming_distance('111', '000'); SELECT hamming_distance(B'111', B'000');
SELECT hamming_distance('10101010101010101010', '01010101010101010101'); SELECT hamming_distance(B'111', B'00');
SELECT hamming_distance('', '');
SELECT hamming_distance('111', '00');
SELECT hamming_distance('111', '000'::varbit(4));
SELECT hamming_distance('111', '0000'::varbit(4));
SELECT jaccard_distance('1111', '1111'); SELECT jaccard_distance(B'1111', B'1111');
SELECT jaccard_distance('1111', '1110'); SELECT jaccard_distance(B'1111', B'1110');
SELECT jaccard_distance('1111', '1100'); SELECT jaccard_distance(B'1111', B'1100');
SELECT jaccard_distance('1111', '1000'); SELECT jaccard_distance(B'1111', B'1000');
SELECT jaccard_distance('1111', '0000'); SELECT jaccard_distance(B'1111', B'0000');
SELECT jaccard_distance('1100', '1000'); SELECT jaccard_distance(B'1100', B'1000');
SELECT jaccard_distance('10101010101010101010', '01010101010101010101'); SELECT jaccard_distance(B'1111', B'000');
SELECT jaccard_distance('', '');
SELECT jaccard_distance('1111', '000');
SELECT jaccard_distance('1111', '0000'::varbit(5));
SELECT jaccard_distance('1111', '00000'::varbit(5));

View File

@@ -11,7 +11,6 @@ SELECT '[65519,-65519]'::halfvec;
SELECT '[65520,-65520]'::halfvec; SELECT '[65520,-65520]'::halfvec;
SELECT '[1e-8,-1e-8]'::halfvec; SELECT '[1e-8,-1e-8]'::halfvec;
SELECT '[4e38,1]'::halfvec; SELECT '[4e38,1]'::halfvec;
SELECT '[1e-46,1]'::halfvec;
SELECT '[1,2,3'::halfvec; SELECT '[1,2,3'::halfvec;
SELECT '[1,2,3]9'::halfvec; SELECT '[1,2,3]9'::halfvec;
SELECT '1,2,3'::halfvec; SELECT '1,2,3'::halfvec;

View File

@@ -10,9 +10,3 @@ SELECT * FROM t ORDER BY val <~> B'111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
DROP TABLE t; DROP TABLE t;
-- TODO move
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
DROP TABLE t;

View File

@@ -1,13 +1,13 @@
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL); INSERT INTO t (val) VALUES ('{}'), ('{0:1,1:2,2:3}'), ('{0:1,1:1,2:1}'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops); CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3'); INSERT INTO t (val) VALUES ('{0:1,1:2,2:4}');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <=> '{0:3,1:3,2:3}';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
DROP TABLE t; DROP TABLE t;

View File

@@ -1,12 +1,12 @@
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL); INSERT INTO t (val) VALUES ('{}'), ('{0:1,1:2,2:3}'), ('{0:1,1:1,2:1}'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops); CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3'); INSERT INTO t (val) VALUES ('{0:1,1:2,2:4}');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <#> '{0:3,1:3,2:3}';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
DROP TABLE t; DROP TABLE t;

View File

@@ -1,22 +1,22 @@
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val sparsevec(3)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL); INSERT INTO t (val) VALUES ('{}'), ('{0:1,1:2,2:3}'), ('{0:1,1:1,2:1}'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops); CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3'); INSERT INTO t (val) VALUES ('{0:1,1:2,2:4}');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <-> '{0:3,1:3,2:3}';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
SELECT COUNT(*) FROM t; SELECT COUNT(*) FROM t;
TRUNCATE t; TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3'; SELECT * FROM t ORDER BY val <-> '{0:3,1:3,2:3}';
DROP TABLE t; DROP TABLE t;
-- TODO move -- TODO move
CREATE TABLE t (val sparsevec(1001)); CREATE TABLE t (val sparsevec);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec); INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops); CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
TRUNCATE t; TRUNCATE t;

View File

@@ -1,13 +1,12 @@
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2'); SELECT l2_distance('{}'::sparsevec, '{0:3,1:4}');
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2'); SELECT l2_distance('{}'::sparsevec, '{1:1}');
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2'; SELECT '{}'::sparsevec <-> '{0:3,1:4}';
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2'); SELECT inner_product('{0:1,1:2}'::sparsevec, '{0:2,1:4}');
SELECT sparsevec_negative_inner_product('{1:1,2:2}/2', '{1:2,2:4}/2'); SELECT sparsevec_negative_inner_product('{0:1,1:2}', '{0:2,1:4}');
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2'); SELECT cosine_distance('{0:1,1:2}'::sparsevec, '{0:2,1:4}');
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2'); SELECT cosine_distance('{0:1,1:2}'::sparsevec, '{}');
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2'); SELECT cosine_distance('{0:1,1:1}'::sparsevec, '{0:-1,1:-1}');
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2'); SELECT cosine_distance('{0:1}'::sparsevec, '{1:2}');
SELECT cosine_distance('{}/1'::sparsevec, '{}/1'); SELECT cosine_distance('{}'::sparsevec, '{}');
SELECT cosine_distance('{1:2}/2'::sparsevec, '{1:1}/3');

View File

@@ -1,19 +1,18 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec; SELECT '{0:1.5,2:3.5}'::sparsevec;
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector; SELECT '{0:1.5,2:3.5}'::sparsevec::vector;
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(5); SELECT '{0:1.5,2:3.5}'::sparsevec::vector(5);
SELECT '{1:1.5,3:3.5}/5'::sparsevec::vector(4); SELECT '{0:1.5,2:3.5}'::sparsevec::vector(4);
SELECT '{0:1.5,2:3.5}'::sparsevec::vector(2);
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec; SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
SELECT '{1:0,2:1,3:0}/3'::sparsevec; SELECT '{0:0,1:1,2:0}'::sparsevec;
SELECT '{2:1,1:1}/2'::sparsevec; SELECT '{1:1,0:1}'::sparsevec;
SELECT '{}/5'::sparsevec; SELECT '{}'::sparsevec;
SELECT '{}/-1'::sparsevec; SELECT '{}'::sparsevec::vector;
SELECT '{}/100001'::sparsevec;
SELECT '{}/16001'::sparsevec::vector;
SELECT '{0:1}/1'::sparsevec; SELECT '{-1:1}'::sparsevec;
SELECT '{2:1}/1'::sparsevec; SELECT '{1:1}'::sparsevec;
SELECT '{}/1'::sparsevec(2); SELECT '{}'::sparsevec(2);

View File

@@ -86,7 +86,7 @@ foreach (@queries)
push(@expected, $res); push(@expected, $res);
} }
test_recall(0.18, $limit, "before vacuum"); test_recall(0.19, $limit, "before vacuum");
test_recall(0.95, 100, "before vacuum"); test_recall(0.95, 100, "before vacuum");
# TODO Test concurrent inserts with vacuum # TODO Test concurrent inserts with vacuum

View File

@@ -99,7 +99,7 @@ for my $i (0 .. $#operators)
)); ));
# Test approximate results # Test approximate results
my $min = $operator eq "<\%>" ? 0.95 : 0.98; my $min = $operator eq "<\%>" ? 0.96 : 0.98;
test_recall($min, $operator); test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;"); $node->safe_psql("postgres", "DROP INDEX idx;");

View File

@@ -1,128 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More;
my $node;
my @queries = ();
my @expected;
my $limit = 20;
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids;
my @expected_ids = split("\n", $expected[$i]);
foreach (@expected_ids)
{
if (exists($actual_set{$_}))
{
$correct++;
}
$total++;
}
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v sparsevec(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()]::vector::sparsevec FROM generate_series(1, 10000) i;"
);
# Generate queries
for (1 .. 20)
{
my $r1 = rand();
my $r2 = rand();
my $r3 = rand();
push(@queries, "{1:$r1,2:$r2,3:$r3}/3");
}
# Check each index type
my @operators = ("<->", "<#>", "<=>");
my @opclasses = ("sparsevec_l2_ops", "sparsevec_ip_ops", "sparsevec_cosine_ops");
for my $i (0 .. $#operators)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Get exact results
@expected = ();
foreach (@queries)
{
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
push(@expected, $res);
}
# Build index serially
$node->safe_psql("postgres", qq(
SET max_parallel_maintenance_workers = 0;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
# Test approximate results
my $min = $operator eq "<#>" ? 0.80 : 0.99;
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel in memory
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET min_parallel_table_scan_size = 1;
CREATE INDEX idx ON tst USING hnsw (v $opclass);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
# Test approximate results
test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;");
# Build index in parallel on disk
# Set parallel_workers on table to use workers with low maintenance_work_mem
($ret, $stdout, $stderr) = $node->psql("postgres", qq(
ALTER TABLE tst SET (parallel_workers = 2);
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '4MB';
CREATE INDEX idx ON tst USING hnsw (v $opclass);
ALTER TABLE tst RESET (parallel_workers);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem/);
$node->safe_psql("postgres", "DROP INDEX idx;");
}
done_testing();