mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 12:07:34 +08:00
Compare commits
1 Commits
half-dispa
...
type-categ
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2dc021d935 |
1
.github/workflows/build.yml
vendored
1
.github/workflows/build.yml
vendored
@@ -125,7 +125,6 @@ jobs:
|
||||
- uses: ankane/setup-postgres-valgrind@v1
|
||||
with:
|
||||
postgres-version: 16
|
||||
check-ub: yes
|
||||
- run: make OPTFLAGS=""
|
||||
- run: sudo --preserve-env=PG_CONFIG make install
|
||||
- run: make installcheck
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
- Added `halfvec` type
|
||||
- Added `sparsevec` type
|
||||
- Added support for indexing `bit` type
|
||||
- Added support for `bit` vectors to HNSW
|
||||
- Added `binary_quantize` function
|
||||
- Added `hamming_distance` function
|
||||
- Added `jaccard_distance` function
|
||||
|
||||
@@ -353,7 +353,6 @@ Supported types are:
|
||||
|
||||
- `vector` - up to 2,000 dimensions
|
||||
- `halfvec` - up to 4,000 dimensions (unreleased)
|
||||
- `bit` - up to 64,000 dimensions (unreleased)
|
||||
|
||||
### Query Options
|
||||
|
||||
|
||||
@@ -33,18 +33,6 @@ CREATE OPERATOR <%> (
|
||||
COMMUTATOR = '<%>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit),
|
||||
FUNCTION 3 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit),
|
||||
FUNCTION 3 jaccard_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
|
||||
@@ -26,7 +26,9 @@ CREATE TYPE vector (
|
||||
TYPMOD_IN = vector_typmod_in,
|
||||
RECEIVE = vector_recv,
|
||||
SEND = vector_send,
|
||||
STORAGE = external
|
||||
STORAGE = external,
|
||||
CATEGORY = 'v',
|
||||
PREFERRED = true
|
||||
);
|
||||
|
||||
-- vector functions
|
||||
@@ -312,8 +314,6 @@ CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
|
||||
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
-- bit operators
|
||||
|
||||
CREATE OPERATOR <~> (
|
||||
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
|
||||
COMMUTATOR = '<~>'
|
||||
@@ -324,20 +324,6 @@ CREATE OPERATOR <%> (
|
||||
COMMUTATOR = '<%>'
|
||||
);
|
||||
|
||||
-- bit opclasses
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 hamming_distance(bit, bit),
|
||||
FUNCTION 3 hamming_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_jaccard_ops
|
||||
FOR TYPE bit USING ivfflat AS
|
||||
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 jaccard_distance(bit, bit),
|
||||
FUNCTION 3 jaccard_distance(bit, bit);
|
||||
|
||||
CREATE OPERATOR CLASS bit_hamming_ops
|
||||
FOR TYPE bit USING hnsw AS
|
||||
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
|
||||
@@ -373,7 +359,8 @@ CREATE TYPE halfvec (
|
||||
TYPMOD_IN = halfvec_typmod_in,
|
||||
RECEIVE = halfvec_recv,
|
||||
SEND = halfvec_send,
|
||||
STORAGE = external
|
||||
STORAGE = external,
|
||||
CATEGORY = 'v'
|
||||
);
|
||||
|
||||
-- halfvec functions
|
||||
@@ -669,7 +656,8 @@ CREATE TYPE sparsevec (
|
||||
TYPMOD_IN = sparsevec_typmod_in,
|
||||
RECEIVE = sparsevec_recv,
|
||||
SEND = sparsevec_send,
|
||||
STORAGE = external
|
||||
STORAGE = external,
|
||||
CATEGORY = 'v'
|
||||
);
|
||||
|
||||
-- sparsevec functions
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
float (*HalfvecL2SquaredDistance) (int dim, half * ax, half * bx);
|
||||
float (*HalfvecInnerProduct) (int dim, half * ax, half * bx);
|
||||
double (*HalfvecCosineSimilarity) (int dim, half * ax, half * bx);
|
||||
@@ -193,18 +192,10 @@ HalfvecCosineSimilarityF16cFma(int dim, half * ax, half * bx)
|
||||
#endif
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
#define CPU_FEATURE_FMA (1 << 12)
|
||||
#define CPU_FEATURE_OSXSAVE (1 << 27)
|
||||
#define CPU_FEATURE_AVX (1 << 28)
|
||||
#define CPU_FEATURE_F16C (1 << 29)
|
||||
#define CPU_FEATURE_FMA (1 << 12)
|
||||
#define CPU_FEATURE_F16C (1 << 29)
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define TARGET_XSAVE
|
||||
#else
|
||||
#define TARGET_XSAVE __attribute__((target("xsave")))
|
||||
#endif
|
||||
|
||||
TARGET_XSAVE static bool
|
||||
static bool
|
||||
SupportsCpuFeature(unsigned int feature)
|
||||
{
|
||||
unsigned int exx[4] = {0, 0, 0, 0};
|
||||
@@ -215,12 +206,6 @@ SupportsCpuFeature(unsigned int feature)
|
||||
__cpuid(exx, 1);
|
||||
#endif
|
||||
|
||||
if ((exx[2] & CPU_FEATURE_OSXSAVE) != CPU_FEATURE_OSXSAVE)
|
||||
return false;
|
||||
|
||||
if ((_xgetbv(0) & 6) != 6)
|
||||
return false;
|
||||
|
||||
return (exx[2] & feature) == feature;
|
||||
}
|
||||
#endif
|
||||
@@ -237,7 +222,7 @@ HalfvecInit(void)
|
||||
HalfvecCosineSimilarity = HalfvecCosineSimilarityDefault;
|
||||
|
||||
#ifdef HALFVEC_DISPATCH
|
||||
if (SupportsCpuFeature(CPU_FEATURE_AVX | CPU_FEATURE_FMA | CPU_FEATURE_F16C))
|
||||
if (SupportsCpuFeature(CPU_FEATURE_FMA | CPU_FEATURE_F16C))
|
||||
{
|
||||
HalfvecL2SquaredDistance = HalfvecL2SquaredDistanceF16cFma;
|
||||
HalfvecInnerProduct = HalfvecInnerProductF16cFma;
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include "access/tableam.h"
|
||||
#include "access/parallel.h"
|
||||
#include "access/xact.h"
|
||||
#include "bitvector.h"
|
||||
#include "catalog/index.h"
|
||||
#include "catalog/pg_operator_d.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
@@ -325,8 +324,6 @@ GetMaxDimensions(IvfflatType type)
|
||||
|
||||
if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
maxDimensions *= 2;
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
maxDimensions *= 32;
|
||||
|
||||
return maxDimensions;
|
||||
}
|
||||
@@ -341,8 +338,6 @@ GetItemSize(IvfflatType type, int dimensions)
|
||||
return VECTOR_SIZE(dimensions);
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
return HALFVEC_SIZE(dimensions);
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
return VARBITTOTALLEN(dimensions);
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
}
|
||||
|
||||
@@ -46,8 +46,7 @@
|
||||
typedef enum IvfflatType
|
||||
{
|
||||
IVFFLAT_TYPE_VECTOR,
|
||||
IVFFLAT_TYPE_HALFVEC,
|
||||
IVFFLAT_TYPE_BIT
|
||||
IVFFLAT_TYPE_HALFVEC
|
||||
} IvfflatType;
|
||||
|
||||
/* Build phases */
|
||||
@@ -264,7 +263,7 @@ typedef struct IvfflatScanOpaqueData
|
||||
|
||||
typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
|
||||
|
||||
#define VECTOR_ARRAY_SIZE(_length, _size) (sizeof(VectorArrayData) + (_length) * MAXALIGN(_size))
|
||||
#define VECTOR_ARRAY_SIZE(_length, _size) (sizeof(VectorArrayData) + (_length) * _size)
|
||||
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) (_arr)->items + (_offset) * (_arr)->itemsize)
|
||||
#define VectorArrayGet(_arr, _offset) VECTOR_ARRAY_OFFSET(_arr, _offset)
|
||||
#define VectorArraySet(_arr, _offset, _val) memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, (_arr)->itemsize)
|
||||
|
||||
@@ -3,12 +3,10 @@
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "bitvector.h"
|
||||
#include "halfutils.h"
|
||||
#include "halfvec.h"
|
||||
#include "ivfflat.h"
|
||||
#include "miscadmin.h"
|
||||
#include "utils/builtins.h"
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "vector.h"
|
||||
@@ -136,15 +134,6 @@ CompareHalfVectors(const void *a, const void *b)
|
||||
return halfvec_cmp_internal((HalfVector *) a, (HalfVector *) b);
|
||||
}
|
||||
|
||||
/*
|
||||
* Compare bit vectors
|
||||
*/
|
||||
static int
|
||||
CompareBitVectors(const void *a, const void *b)
|
||||
{
|
||||
return DirectFunctionCall2(bitcmp, VarBitPGetDatum((VarBit *) a), VarBitPGetDatum((VarBit *) b));
|
||||
}
|
||||
|
||||
/*
|
||||
* Quick approach if we have little data
|
||||
*/
|
||||
@@ -162,8 +151,6 @@ QuickCenters(Relation index, VectorArray samples, VectorArray centers, IvfflatTy
|
||||
qsort(samples->items, samples->length, samples->itemsize, CompareVectors);
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
qsort(samples->items, samples->length, samples->itemsize, CompareHalfVectors);
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
qsort(samples->items, samples->length, samples->itemsize, CompareBitVectors);
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
@@ -204,16 +191,6 @@ QuickCenters(Relation index, VectorArray samples, VectorArray centers, IvfflatTy
|
||||
for (int j = 0; j < dimensions; j++)
|
||||
vec->x[j] = Float4ToHalfUnchecked((float) RandomDouble());
|
||||
}
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
{
|
||||
VarBit *vec = DatumGetVarBitP(center);
|
||||
|
||||
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
|
||||
VARBITLEN(vec) = dimensions;
|
||||
|
||||
for (int j = 0; j < dimensions; j++)
|
||||
VARBITS(vec)[j / dimensions] |= (RandomDouble() > 0.5 ? 1 : 0) << (7 - (j % 8));
|
||||
}
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
@@ -286,17 +263,6 @@ ComputeNewCenters(VectorArray samples, VectorArray aggCenters, VectorArray newCe
|
||||
aggCenter->x[k] += HalfToFloat4(vec->x[k]);
|
||||
}
|
||||
}
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
{
|
||||
for (int j = 0; j < numSamples; j++)
|
||||
{
|
||||
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, closestCenters[j]);
|
||||
VarBit *vec = (VarBit *) VectorArrayGet(samples, j);
|
||||
|
||||
for (int k = 0; k < dimensions; k++)
|
||||
aggCenter->x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
|
||||
}
|
||||
}
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
@@ -342,21 +308,6 @@ ComputeNewCenters(VectorArray samples, VectorArray aggCenters, VectorArray newCe
|
||||
newCenter->x[k] = Float4ToHalfUnchecked(aggCenter->x[k]);
|
||||
}
|
||||
}
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
{
|
||||
for (int j = 0; j < numCenters; j++)
|
||||
{
|
||||
Vector *aggCenter = (Vector *) VectorArrayGet(aggCenters, j);
|
||||
VarBit *newCenter = (VarBit *) VectorArrayGet(newCenters, j);
|
||||
unsigned char *nx = VARBITS(newCenter);
|
||||
|
||||
for (uint32 k = 0; k < VARBITBYTES(newCenter); k++)
|
||||
nx[k] = 0;
|
||||
|
||||
for (int k = 0; k < dimensions; k++)
|
||||
nx[k / 8] |= (aggCenter->x[k] > 0.5) << (7 - (k % 8));
|
||||
}
|
||||
}
|
||||
|
||||
/* Normalize if needed */
|
||||
if (normprocinfo != NULL)
|
||||
@@ -474,18 +425,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, IvfflatTyp
|
||||
vec->dim = dimensions;
|
||||
}
|
||||
}
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
{
|
||||
newCenters = VectorArrayInit(numCenters, dimensions, centers->itemsize);
|
||||
|
||||
for (int j = 0; j < numCenters; j++)
|
||||
{
|
||||
VarBit *vec = (VarBit *) VectorArrayGet(newCenters, j);
|
||||
|
||||
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
|
||||
VARBITLEN(vec) = dimensions;
|
||||
}
|
||||
}
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
@@ -703,26 +642,23 @@ CheckCenters(Relation index, VectorArray centers, IvfflatType type)
|
||||
elog(ERROR, "Infinite value detected. Please report a bug.");
|
||||
}
|
||||
}
|
||||
else if (type != IVFFLAT_TYPE_BIT)
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
}
|
||||
|
||||
if (type != IVFFLAT_TYPE_BIT)
|
||||
{
|
||||
/* Ensure no duplicate centers */
|
||||
/* Fine to sort in-place */
|
||||
if (type == IVFFLAT_TYPE_VECTOR)
|
||||
qsort(centers->items, centers->length, centers->itemsize, CompareVectors);
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
qsort(centers->items, centers->length, centers->itemsize, CompareHalfVectors);
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
/* Ensure no duplicate centers */
|
||||
/* Fine to sort in-place */
|
||||
if (type == IVFFLAT_TYPE_VECTOR)
|
||||
qsort(centers->items, centers->length, centers->itemsize, CompareVectors);
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
qsort(centers->items, centers->length, centers->itemsize, CompareHalfVectors);
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
|
||||
for (int i = 1; i < centers->length; i++)
|
||||
{
|
||||
if (datumIsEqual(PointerGetDatum(VectorArrayGet(centers, i)), PointerGetDatum(VectorArrayGet(centers, i - 1)), false, -1))
|
||||
elog(ERROR, "Duplicate centers detected. Please report a bug.");
|
||||
}
|
||||
for (int i = 1; i < centers->length; i++)
|
||||
{
|
||||
if (datumIsEqual(PointerGetDatum(VectorArrayGet(centers, i)), PointerGetDatum(VectorArrayGet(centers, i - 1)), false, -1))
|
||||
elog(ERROR, "Duplicate centers detected. Please report a bug.");
|
||||
}
|
||||
|
||||
/* Ensure no zero vectors for cosine distance */
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
#include <float.h>
|
||||
|
||||
#include "access/relscan.h"
|
||||
#include "bitvector.h"
|
||||
#include "catalog/pg_operator_d.h"
|
||||
#include "catalog/pg_type_d.h"
|
||||
#include "halfvec.h"
|
||||
@@ -196,8 +195,6 @@ GetScanValue(IndexScanDesc scan)
|
||||
value = PointerGetDatum(InitVector(so->dimensions));
|
||||
else if (type == IVFFLAT_TYPE_HALFVEC)
|
||||
value = PointerGetDatum(InitHalfVector(so->dimensions));
|
||||
else if (type == IVFFLAT_TYPE_BIT)
|
||||
value = PointerGetDatum(InitBitVector(so->dimensions));
|
||||
else
|
||||
elog(ERROR, "Unsupported type");
|
||||
}
|
||||
|
||||
@@ -18,9 +18,6 @@ VectorArrayInit(int maxlen, int dimensions, Size itemsize)
|
||||
{
|
||||
VectorArray res = palloc(sizeof(VectorArrayData));
|
||||
|
||||
/* Ensure items are aligned to prevent UB */
|
||||
itemsize = MAXALIGN(itemsize);
|
||||
|
||||
res->length = 0;
|
||||
res->maxlen = maxlen;
|
||||
res->dim = dimensions;
|
||||
@@ -76,9 +73,6 @@ IvfflatGetType(Relation index)
|
||||
Form_pg_type type;
|
||||
IvfflatType result;
|
||||
|
||||
if (typid == BITOID)
|
||||
return IVFFLAT_TYPE_BIT;
|
||||
|
||||
tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typid));
|
||||
if (!HeapTupleIsValid(tuple))
|
||||
elog(ERROR, "cache lookup failed for type %u", typid);
|
||||
|
||||
@@ -34,14 +34,7 @@
|
||||
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
|
||||
|
||||
/* target_clones requires glibc */
|
||||
#if defined(__gnu_linux__) && defined(__has_attribute)
|
||||
/* Use separate line for portability */
|
||||
#if __has_attribute(target_clones)
|
||||
#define HAVE_TARGET_CLONES
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__x86_64__) && defined(HAVE_TARGET_CLONES) && !defined(__FMA__)
|
||||
#if defined(__x86_64__) && defined(__gnu_linux__) && defined(__has_attribute) && __has_attribute(target_clones) && !defined(__FMA__)
|
||||
#define VECTOR_DISPATCH __attribute__((target_clones("default", "fma")))
|
||||
#else
|
||||
#define VECTOR_DISPATCH
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val bit(3));
|
||||
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES (B'110');
|
||||
SELECT * FROM t ORDER BY val <~> B'111';
|
||||
val
|
||||
-----
|
||||
111
|
||||
110
|
||||
100
|
||||
000
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
-- TODO move
|
||||
CREATE TABLE t (val varbit(3));
|
||||
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
|
||||
ERROR: type not supported for ivfflat index
|
||||
CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1);
|
||||
NOTICE: ivfflat index created with little data
|
||||
DETAIL: This will cause low recall.
|
||||
HINT: Drop the index until the table has more data.
|
||||
CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
|
||||
ERROR: column cannot have more than 64000 dimensions for ivfflat index
|
||||
CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
|
||||
NOTICE: ivfflat index created with little data
|
||||
DETAIL: This will cause low recall.
|
||||
HINT: Drop the index until the table has more data.
|
||||
DROP TABLE t;
|
||||
@@ -1,21 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
CREATE TABLE t (val bit(4));
|
||||
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val bit_jaccard_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES (B'1110');
|
||||
SELECT * FROM t ORDER BY val <%> B'1111';
|
||||
val
|
||||
------
|
||||
1111
|
||||
1110
|
||||
1100
|
||||
0000
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -1,20 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val bit(3));
|
||||
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
|
||||
|
||||
INSERT INTO t (val) VALUES (B'110');
|
||||
|
||||
SELECT * FROM t ORDER BY val <~> B'111';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
-- TODO move
|
||||
CREATE TABLE t (val varbit(3));
|
||||
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
|
||||
CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1);
|
||||
CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
|
||||
CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
|
||||
DROP TABLE t;
|
||||
@@ -1,12 +0,0 @@
|
||||
SET enable_seqscan = off;
|
||||
|
||||
CREATE TABLE t (val bit(4));
|
||||
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
|
||||
CREATE INDEX ON t USING ivfflat (val bit_jaccard_ops) WITH (lists = 1);
|
||||
|
||||
INSERT INTO t (val) VALUES (B'1110');
|
||||
|
||||
SELECT * FROM t ORDER BY val <%> B'1111';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -105,8 +105,8 @@ for my $i (0 .. $#operators)
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
test_recall(1, 0.34, $operator);
|
||||
test_recall(10, 0.93, $operator);
|
||||
test_recall(1, 0.35, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
|
||||
# Test probes equals lists
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
my $dim = 52;
|
||||
my $max = 2**$dim;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($probes, $min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator $queries[0] LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx on tst/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
SELECT i FROM tst ORDER BY v $operator $queries[$i] LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
my %expected_set = map { $_ => 1 } @expected_ids;
|
||||
|
||||
foreach (@actual_ids)
|
||||
{
|
||||
if (exists($expected_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
}
|
||||
|
||||
$total += $limit;
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v bit($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT i, (random() * $max)::bigint::bit($dim) FROM generate_series(1, 100000) i;"
|
||||
);
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r = int(rand() * $max);
|
||||
push(@queries, "${r}::bigint::bit($dim)");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<~>", "<\%>");
|
||||
my @opclasses = ("bit_hamming_ops", "bit_jaccard_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
WITH top AS (
|
||||
SELECT v $operator $_ AS distance FROM tst ORDER BY distance LIMIT $limit
|
||||
)
|
||||
SELECT i FROM tst WHERE (v $operator $_) <= (SELECT MAX(distance) FROM top)
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Build index serially
|
||||
$node->safe_psql("postgres", qq(
|
||||
SET max_parallel_maintenance_workers = 0;
|
||||
CREATE INDEX idx ON tst USING ivfflat (v $opclass);
|
||||
));
|
||||
|
||||
# Test approximate results
|
||||
test_recall(1, 0.08, $operator);
|
||||
test_recall(10, 0.50, $operator);
|
||||
|
||||
# Test probes equals lists
|
||||
test_recall(100, 1.00, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
|
||||
# Build index in parallel
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET client_min_messages = DEBUG;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
CREATE INDEX idx ON tst USING ivfflat (v $opclass);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
|
||||
# Test approximate results
|
||||
test_recall(1, 0.08, $operator);
|
||||
test_recall(10, 0.50, $operator);
|
||||
|
||||
# Test probes equals lists
|
||||
test_recall(100, 1.00, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
Reference in New Issue
Block a user