mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 20:15:46 +08:00
Compare commits
18 Commits
v0.6.2
...
vector-ran
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4f4286f74e | ||
|
|
11ea3d8483 | ||
|
|
2c48e3edc2 | ||
|
|
7d63bb4b98 | ||
|
|
de410a2915 | ||
|
|
64aa99aa31 | ||
|
|
997fa167da | ||
|
|
67eec4edbf | ||
|
|
396090d8e0 | ||
|
|
ba18942fcf | ||
|
|
8e59455c3c | ||
|
|
bd50e3067d | ||
|
|
af9d4ad659 | ||
|
|
08abb63cbe | ||
|
|
06b8556a49 | ||
|
|
3f674c9994 | ||
|
|
31e41b3ba9 | ||
|
|
903a925662 |
34
README.md
34
README.md
@@ -415,7 +415,19 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python
|
|||||||
|
|
||||||
### Tuning
|
### Tuning
|
||||||
|
|
||||||
Use a tool like [PgTune](https://pgtune.leopard.in.ua/) to set initial values for Postgres server parameters.
|
Use a tool like [PgTune](https://pgtune.leopard.in.ua/) to set initial values for Postgres server parameters. For instance, `shared_buffers` should typically be 25% of the server’s memory. You can find the config file with:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SHOW config_file;
|
||||||
|
```
|
||||||
|
|
||||||
|
And check individual settings with:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SHOW shared_buffers;
|
||||||
|
```
|
||||||
|
|
||||||
|
Be sure to restart Postgres for changes to take effect.
|
||||||
|
|
||||||
### Loading
|
### Loading
|
||||||
|
|
||||||
@@ -604,6 +616,18 @@ and query with:
|
|||||||
SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5;
|
SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5;
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Are binary vectors supported?
|
||||||
|
|
||||||
|
You can store binary vectors and perform exact nearest neighbor search by Hamming distance in Postgres without an extension ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py)).
|
||||||
|
|
||||||
|
```tsql
|
||||||
|
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
|
||||||
|
INSERT INTO items (embedding) VALUES (B'000'), (B'111');
|
||||||
|
SELECT * FROM items ORDER BY bit_count(embedding # B'101') LIMIT 5;
|
||||||
|
```
|
||||||
|
|
||||||
|
Indexing is not currently supported.
|
||||||
|
|
||||||
#### Do indexes need to fit into memory?
|
#### Do indexes need to fit into memory?
|
||||||
|
|
||||||
No, but like other index types, you’ll likely see better performance if they do. You can get the size of an index with:
|
No, but like other index types, you’ll likely see better performance if they do. You can get the size of an index with:
|
||||||
@@ -659,6 +683,8 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
|
|||||||
|
|
||||||
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
|
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
|
||||||
|
|
||||||
|
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
|
||||||
|
|
||||||
#### Why are there less results for a query after adding an IVFFlat index?
|
#### Why are there less results for a query after adding an IVFFlat index?
|
||||||
|
|
||||||
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
||||||
@@ -669,11 +695,13 @@ DROP INDEX index_name;
|
|||||||
|
|
||||||
Results can also be limited by the number of probes (`ivfflat.probes`).
|
Results can also be limited by the number of probes (`ivfflat.probes`).
|
||||||
|
|
||||||
|
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
|
||||||
|
|
||||||
## Reference
|
## Reference
|
||||||
|
|
||||||
### Vector Type
|
### Vector Type
|
||||||
|
|
||||||
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
|
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single-precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
|
||||||
|
|
||||||
### Vector Operators
|
### Vector Operators
|
||||||
|
|
||||||
@@ -697,7 +725,7 @@ l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
|
|||||||
vector_dims(vector) → integer | number of dimensions |
|
vector_dims(vector) → integer | number of dimensions |
|
||||||
vector_norm(vector) → double precision | Euclidean norm |
|
vector_norm(vector) → double precision | Euclidean norm |
|
||||||
|
|
||||||
### Aggregate Functions
|
### Vector Aggregate Functions
|
||||||
|
|
||||||
Function | Description | Added
|
Function | Description | Added
|
||||||
--- | --- | ---
|
--- | --- | ---
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||||
\echo Use "CREATE EXTENSION vector" to load this file. \quit
|
\echo Use "CREATE EXTENSION vector" to load this file. \quit
|
||||||
|
|
||||||
-- type
|
-- vector type
|
||||||
|
|
||||||
CREATE TYPE vector;
|
CREATE TYPE vector;
|
||||||
|
|
||||||
@@ -29,7 +29,7 @@ CREATE TYPE vector (
|
|||||||
STORAGE = external
|
STORAGE = external
|
||||||
);
|
);
|
||||||
|
|
||||||
-- functions
|
-- vector functions
|
||||||
|
|
||||||
CREATE FUNCTION l2_distance(vector, vector) RETURNS float8
|
CREATE FUNCTION l2_distance(vector, vector) RETURNS float8
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
@@ -58,7 +58,7 @@ CREATE FUNCTION vector_sub(vector, vector) RETURNS vector
|
|||||||
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
|
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
|
|
||||||
-- private functions
|
-- vector private functions
|
||||||
|
|
||||||
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
|
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
@@ -99,7 +99,7 @@ CREATE FUNCTION vector_avg(double precision[]) RETURNS vector
|
|||||||
CREATE FUNCTION vector_combine(double precision[], double precision[]) RETURNS double precision[]
|
CREATE FUNCTION vector_combine(double precision[], double precision[]) RETURNS double precision[]
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
|
|
||||||
-- aggregates
|
-- vector aggregates
|
||||||
|
|
||||||
CREATE AGGREGATE avg(vector) (
|
CREATE AGGREGATE avg(vector) (
|
||||||
SFUNC = vector_accum,
|
SFUNC = vector_accum,
|
||||||
@@ -117,7 +117,7 @@ CREATE AGGREGATE sum(vector) (
|
|||||||
PARALLEL = SAFE
|
PARALLEL = SAFE
|
||||||
);
|
);
|
||||||
|
|
||||||
-- cast functions
|
-- vector cast functions
|
||||||
|
|
||||||
CREATE FUNCTION vector(vector, integer, boolean) RETURNS vector
|
CREATE FUNCTION vector(vector, integer, boolean) RETURNS vector
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
@@ -137,7 +137,7 @@ CREATE FUNCTION array_to_vector(numeric[], integer, boolean) RETURNS vector
|
|||||||
CREATE FUNCTION vector_to_float4(vector, integer, boolean) RETURNS real[]
|
CREATE FUNCTION vector_to_float4(vector, integer, boolean) RETURNS real[]
|
||||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||||
|
|
||||||
-- casts
|
-- vector casts
|
||||||
|
|
||||||
CREATE CAST (vector AS vector)
|
CREATE CAST (vector AS vector)
|
||||||
WITH FUNCTION vector(vector, integer, boolean) AS IMPLICIT;
|
WITH FUNCTION vector(vector, integer, boolean) AS IMPLICIT;
|
||||||
@@ -157,7 +157,7 @@ CREATE CAST (double precision[] AS vector)
|
|||||||
CREATE CAST (numeric[] AS vector)
|
CREATE CAST (numeric[] AS vector)
|
||||||
WITH FUNCTION array_to_vector(numeric[], integer, boolean) AS ASSIGNMENT;
|
WITH FUNCTION array_to_vector(numeric[], integer, boolean) AS ASSIGNMENT;
|
||||||
|
|
||||||
-- operators
|
-- vector operators
|
||||||
|
|
||||||
CREATE OPERATOR <-> (
|
CREATE OPERATOR <-> (
|
||||||
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l2_distance,
|
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l2_distance,
|
||||||
@@ -240,7 +240,7 @@ CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnswhandler;
|
|||||||
|
|
||||||
COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
|
COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
|
||||||
|
|
||||||
-- opclasses
|
-- vector opclasses
|
||||||
|
|
||||||
CREATE OPERATOR CLASS vector_ops
|
CREATE OPERATOR CLASS vector_ops
|
||||||
DEFAULT FOR TYPE vector USING btree AS
|
DEFAULT FOR TYPE vector USING btree AS
|
||||||
|
|||||||
10
src/hnsw.h
10
src/hnsw.h
@@ -55,6 +55,11 @@
|
|||||||
#define HNSW_UPDATE_ENTRY_GREATER 1
|
#define HNSW_UPDATE_ENTRY_GREATER 1
|
||||||
#define HNSW_UPDATE_ENTRY_ALWAYS 2
|
#define HNSW_UPDATE_ENTRY_ALWAYS 2
|
||||||
|
|
||||||
|
typedef enum HnswType
|
||||||
|
{
|
||||||
|
HNSW_TYPE_VECTOR
|
||||||
|
} HnswType;
|
||||||
|
|
||||||
/* Build phases */
|
/* Build phases */
|
||||||
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
|
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
|
||||||
#define PROGRESS_HNSW_PHASE_LOAD 2
|
#define PROGRESS_HNSW_PHASE_LOAD 2
|
||||||
@@ -242,6 +247,7 @@ typedef struct HnswBuildState
|
|||||||
Relation index;
|
Relation index;
|
||||||
IndexInfo *indexInfo;
|
IndexInfo *indexInfo;
|
||||||
ForkNumber forkNum;
|
ForkNumber forkNum;
|
||||||
|
HnswType type;
|
||||||
|
|
||||||
/* Settings */
|
/* Settings */
|
||||||
int dimensions;
|
int dimensions;
|
||||||
@@ -262,7 +268,6 @@ typedef struct HnswBuildState
|
|||||||
HnswGraph *graph;
|
HnswGraph *graph;
|
||||||
double ml;
|
double ml;
|
||||||
int maxLevel;
|
int maxLevel;
|
||||||
Vector *normvec;
|
|
||||||
|
|
||||||
/* Memory */
|
/* Memory */
|
||||||
MemoryContext graphCtx;
|
MemoryContext graphCtx;
|
||||||
@@ -367,7 +372,8 @@ typedef struct HnswVacuumState
|
|||||||
int HnswGetM(Relation index);
|
int HnswGetM(Relation index);
|
||||||
int HnswGetEfConstruction(Relation index);
|
int HnswGetEfConstruction(Relation index);
|
||||||
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
|
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
|
||||||
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
|
HnswType HnswGetType(Relation index);
|
||||||
|
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type);
|
||||||
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
||||||
void HnswInitPage(Buffer buf, Page page);
|
void HnswInitPage(Buffer buf, Page page);
|
||||||
void HnswInit(void);
|
void HnswInit(void);
|
||||||
|
|||||||
@@ -489,7 +489,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
|||||||
/* Normalize if needed */
|
/* Normalize if needed */
|
||||||
if (buildstate->normprocinfo != NULL)
|
if (buildstate->normprocinfo != NULL)
|
||||||
{
|
{
|
||||||
if (!HnswNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->normvec))
|
if (!HnswNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->type))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -671,10 +671,13 @@ HnswSharedMemoryAlloc(Size size, void *state)
|
|||||||
static void
|
static void
|
||||||
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
|
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
|
||||||
{
|
{
|
||||||
|
int maxDimensions = HNSW_MAX_DIM;
|
||||||
|
|
||||||
buildstate->heap = heap;
|
buildstate->heap = heap;
|
||||||
buildstate->index = index;
|
buildstate->index = index;
|
||||||
buildstate->indexInfo = indexInfo;
|
buildstate->indexInfo = indexInfo;
|
||||||
buildstate->forkNum = forkNum;
|
buildstate->forkNum = forkNum;
|
||||||
|
buildstate->type = HnswGetType(index);
|
||||||
|
|
||||||
buildstate->m = HnswGetM(index);
|
buildstate->m = HnswGetM(index);
|
||||||
buildstate->efConstruction = HnswGetEfConstruction(index);
|
buildstate->efConstruction = HnswGetEfConstruction(index);
|
||||||
@@ -684,8 +687,8 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
|||||||
if (buildstate->dimensions < 0)
|
if (buildstate->dimensions < 0)
|
||||||
elog(ERROR, "column does not have dimensions");
|
elog(ERROR, "column does not have dimensions");
|
||||||
|
|
||||||
if (buildstate->dimensions > HNSW_MAX_DIM)
|
if (buildstate->dimensions > maxDimensions)
|
||||||
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", HNSW_MAX_DIM);
|
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", maxDimensions);
|
||||||
|
|
||||||
if (buildstate->efConstruction < 2 * buildstate->m)
|
if (buildstate->efConstruction < 2 * buildstate->m)
|
||||||
elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
|
elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
|
||||||
@@ -703,9 +706,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
|||||||
buildstate->ml = HnswGetMl(buildstate->m);
|
buildstate->ml = HnswGetMl(buildstate->m);
|
||||||
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
|
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
|
||||||
|
|
||||||
/* Reuse for each tuple */
|
|
||||||
buildstate->normvec = InitVector(buildstate->dimensions);
|
|
||||||
|
|
||||||
buildstate->graphCtx = GenerationContextCreate(CurrentMemoryContext,
|
buildstate->graphCtx = GenerationContextCreate(CurrentMemoryContext,
|
||||||
"Hnsw build graph context",
|
"Hnsw build graph context",
|
||||||
#if PG_VERSION_NUM >= 150000
|
#if PG_VERSION_NUM >= 150000
|
||||||
@@ -729,7 +729,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
|
|||||||
static void
|
static void
|
||||||
FreeBuildState(HnswBuildState * buildstate)
|
FreeBuildState(HnswBuildState * buildstate)
|
||||||
{
|
{
|
||||||
pfree(buildstate->normvec);
|
|
||||||
MemoryContextDelete(buildstate->graphCtx);
|
MemoryContextDelete(buildstate->graphCtx);
|
||||||
MemoryContextDelete(buildstate->tmpCtx);
|
MemoryContextDelete(buildstate->tmpCtx);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -622,7 +622,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
|
|||||||
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
|
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
|
||||||
if (normprocinfo != NULL)
|
if (normprocinfo != NULL)
|
||||||
{
|
{
|
||||||
if (!HnswNormValue(normprocinfo, collation, &value, NULL))
|
if (!HnswNormValue(normprocinfo, collation, &value, HnswGetType(index)))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -40,29 +40,6 @@ GetScanItems(IndexScanDesc scan, Datum q)
|
|||||||
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
|
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Get dimensions from metapage
|
|
||||||
*/
|
|
||||||
static int
|
|
||||||
GetDimensions(Relation index)
|
|
||||||
{
|
|
||||||
Buffer buf;
|
|
||||||
Page page;
|
|
||||||
HnswMetaPage metap;
|
|
||||||
int dimensions;
|
|
||||||
|
|
||||||
buf = ReadBuffer(index, HNSW_METAPAGE_BLKNO);
|
|
||||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
|
||||||
page = BufferGetPage(buf);
|
|
||||||
metap = HnswPageGetMeta(page);
|
|
||||||
|
|
||||||
dimensions = metap->dimensions;
|
|
||||||
|
|
||||||
UnlockReleaseBuffer(buf);
|
|
||||||
|
|
||||||
return dimensions;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Get scan value
|
* Get scan value
|
||||||
*/
|
*/
|
||||||
@@ -73,7 +50,7 @@ GetScanValue(IndexScanDesc scan)
|
|||||||
Datum value;
|
Datum value;
|
||||||
|
|
||||||
if (scan->orderByData->sk_flags & SK_ISNULL)
|
if (scan->orderByData->sk_flags & SK_ISNULL)
|
||||||
value = PointerGetDatum(InitVector(GetDimensions(scan->indexRelation)));
|
value = PointerGetDatum(NULL);
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
value = scan->orderByData->sk_argument;
|
value = scan->orderByData->sk_argument;
|
||||||
@@ -84,7 +61,7 @@ GetScanValue(IndexScanDesc scan)
|
|||||||
|
|
||||||
/* Fine if normalization fails */
|
/* Fine if normalization fails */
|
||||||
if (so->normprocinfo != NULL)
|
if (so->normprocinfo != NULL)
|
||||||
HnswNormValue(so->normprocinfo, so->collation, &value, NULL);
|
HnswNormValue(so->normprocinfo, so->collation, &value, HnswGetType(scan->indexRelation));
|
||||||
}
|
}
|
||||||
|
|
||||||
return value;
|
return value;
|
||||||
|
|||||||
@@ -149,6 +149,15 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
|
|||||||
return index_getprocinfo(index, 1, procnum);
|
return index_getprocinfo(index, 1, procnum);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Get vector type
|
||||||
|
*/
|
||||||
|
HnswType
|
||||||
|
HnswGetType(Relation index)
|
||||||
|
{
|
||||||
|
return HNSW_TYPE_VECTOR;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Divide by the norm
|
* Divide by the norm
|
||||||
*
|
*
|
||||||
@@ -158,21 +167,25 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
|
|||||||
* if it's different than the original value
|
* if it's different than the original value
|
||||||
*/
|
*/
|
||||||
bool
|
bool
|
||||||
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result)
|
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type)
|
||||||
{
|
{
|
||||||
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
|
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
|
||||||
|
|
||||||
if (norm > 0)
|
if (norm > 0)
|
||||||
{
|
{
|
||||||
Vector *v = DatumGetVector(*value);
|
/* TODO Remove vector-specific code */
|
||||||
|
if (type == HNSW_TYPE_VECTOR)
|
||||||
|
{
|
||||||
|
Vector *v = DatumGetVector(*value);
|
||||||
|
Vector *result = InitVector(v->dim);
|
||||||
|
|
||||||
if (result == NULL)
|
for (int i = 0; i < v->dim; i++)
|
||||||
result = InitVector(v->dim);
|
result->x[i] = v->x[i] / norm;
|
||||||
|
|
||||||
for (int i = 0; i < v->dim; i++)
|
*value = PointerGetDatum(result);
|
||||||
result->x[i] = v->x[i] / norm;
|
}
|
||||||
|
else
|
||||||
*value = PointerGetDatum(result);
|
elog(ERROR, "Unsupported type");
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -575,7 +588,12 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
|
|||||||
|
|
||||||
/* Calculate distance */
|
/* Calculate distance */
|
||||||
if (distance != NULL)
|
if (distance != NULL)
|
||||||
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
|
{
|
||||||
|
if (DatumGetPointer(*q) == NULL)
|
||||||
|
*distance = 0;
|
||||||
|
else
|
||||||
|
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
|
||||||
|
}
|
||||||
|
|
||||||
UnlockReleaseBuffer(buf);
|
UnlockReleaseBuffer(buf);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
|
|||||||
*/
|
*/
|
||||||
if (buildstate->kmeansnormprocinfo != NULL)
|
if (buildstate->kmeansnormprocinfo != NULL)
|
||||||
{
|
{
|
||||||
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value, buildstate->normvec))
|
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,7 +105,7 @@ SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
|
|||||||
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
|
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
|
||||||
|
|
||||||
/* Add sample */
|
/* Add sample */
|
||||||
AddSample(values, state);
|
AddSample(values, buildstate);
|
||||||
|
|
||||||
/* Reset memory context */
|
/* Reset memory context */
|
||||||
MemoryContextSwitchTo(oldCtx);
|
MemoryContextSwitchTo(oldCtx);
|
||||||
@@ -153,7 +153,7 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
|
|||||||
/* Normalize if needed */
|
/* Normalize if needed */
|
||||||
if (buildstate->normprocinfo != NULL)
|
if (buildstate->normprocinfo != NULL)
|
||||||
{
|
{
|
||||||
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->normvec))
|
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -356,9 +356,6 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
|
|||||||
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions);
|
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions);
|
||||||
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
|
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
|
||||||
|
|
||||||
/* Reuse for each tuple */
|
|
||||||
buildstate->normvec = InitVector(buildstate->dimensions);
|
|
||||||
|
|
||||||
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
|
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
|
||||||
"Ivfflat build temporary context",
|
"Ivfflat build temporary context",
|
||||||
ALLOCSET_DEFAULT_SIZES);
|
ALLOCSET_DEFAULT_SIZES);
|
||||||
@@ -380,7 +377,6 @@ FreeBuildState(IvfflatBuildState * buildstate)
|
|||||||
{
|
{
|
||||||
VectorArrayFree(buildstate->centers);
|
VectorArrayFree(buildstate->centers);
|
||||||
pfree(buildstate->listInfo);
|
pfree(buildstate->listInfo);
|
||||||
pfree(buildstate->normvec);
|
|
||||||
|
|
||||||
#ifdef IVFFLAT_KMEANS_DEBUG
|
#ifdef IVFFLAT_KMEANS_DEBUG
|
||||||
pfree(buildstate->listSums);
|
pfree(buildstate->listSums);
|
||||||
|
|||||||
@@ -172,7 +172,6 @@ typedef struct IvfflatBuildState
|
|||||||
VectorArray samples;
|
VectorArray samples;
|
||||||
VectorArray centers;
|
VectorArray centers;
|
||||||
ListInfo *listInfo;
|
ListInfo *listInfo;
|
||||||
Vector *normvec;
|
|
||||||
|
|
||||||
#ifdef IVFFLAT_KMEANS_DEBUG
|
#ifdef IVFFLAT_KMEANS_DEBUG
|
||||||
double inertia;
|
double inertia;
|
||||||
@@ -267,7 +266,7 @@ void VectorArrayFree(VectorArray arr);
|
|||||||
void PrintVectorArray(char *msg, VectorArray arr);
|
void PrintVectorArray(char *msg, VectorArray arr);
|
||||||
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
|
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
|
||||||
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
|
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
|
||||||
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
|
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value);
|
||||||
int IvfflatGetLists(Relation index);
|
int IvfflatGetLists(Relation index);
|
||||||
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
|
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
|
||||||
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
|
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
|
|||||||
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
||||||
if (normprocinfo != NULL)
|
if (normprocinfo != NULL)
|
||||||
{
|
{
|
||||||
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value, NULL))
|
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -293,7 +293,7 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
|
|||||||
|
|
||||||
/* Fine if normalization fails */
|
/* Fine if normalization fails */
|
||||||
if (so->normprocinfo != NULL)
|
if (so->normprocinfo != NULL)
|
||||||
IvfflatNormValue(so->normprocinfo, so->collation, &value, NULL);
|
IvfflatNormValue(so->normprocinfo, so->collation, &value);
|
||||||
}
|
}
|
||||||
|
|
||||||
IvfflatBench("GetScanLists", GetScanLists(scan, value));
|
IvfflatBench("GetScanLists", GetScanLists(scan, value));
|
||||||
|
|||||||
@@ -75,16 +75,14 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
|
|||||||
* if it's different than the original value
|
* if it's different than the original value
|
||||||
*/
|
*/
|
||||||
bool
|
bool
|
||||||
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result)
|
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value)
|
||||||
{
|
{
|
||||||
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
|
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
|
||||||
|
|
||||||
if (norm > 0)
|
if (norm > 0)
|
||||||
{
|
{
|
||||||
Vector *v = DatumGetVector(*value);
|
Vector *v = DatumGetVector(*value);
|
||||||
|
Vector *result = InitVector(v->dim);
|
||||||
if (result == NULL)
|
|
||||||
result = InitVector(v->dim);
|
|
||||||
|
|
||||||
for (int i = 0; i < v->dim; i++)
|
for (int i = 0; i < v->dim; i++)
|
||||||
result->x[i] = v->x[i] / norm;
|
result->x[i] = v->x[i] / norm;
|
||||||
|
|||||||
16
src/vector.c
16
src/vector.c
@@ -197,6 +197,8 @@ vector_in(PG_FUNCTION_ARGS)
|
|||||||
|
|
||||||
while (pt != NULL && *stringEnd != ']')
|
while (pt != NULL && *stringEnd != ']')
|
||||||
{
|
{
|
||||||
|
float val;
|
||||||
|
|
||||||
if (dim == VECTOR_MAX_DIM)
|
if (dim == VECTOR_MAX_DIM)
|
||||||
ereport(ERROR,
|
ereport(ERROR,
|
||||||
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
||||||
@@ -212,15 +214,23 @@ vector_in(PG_FUNCTION_ARGS)
|
|||||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||||
|
|
||||||
/* Use strtof like float4in to avoid a double-rounding problem */
|
/* Use strtof like float4in to avoid a double-rounding problem */
|
||||||
x[dim] = strtof(pt, &stringEnd);
|
errno = 0;
|
||||||
CheckElement(x[dim]);
|
val = strtof(pt, &stringEnd);
|
||||||
dim++;
|
|
||||||
|
|
||||||
if (stringEnd == pt)
|
if (stringEnd == pt)
|
||||||
ereport(ERROR,
|
ereport(ERROR,
|
||||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||||
|
|
||||||
|
/* Check for range error like float4in */
|
||||||
|
if (errno == ERANGE && (val == 0 || isinf(val)))
|
||||||
|
ereport(ERROR,
|
||||||
|
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||||
|
errmsg("\"%s\" is out of range for type vector", pt)));
|
||||||
|
|
||||||
|
CheckElement(val);
|
||||||
|
x[dim++] = val;
|
||||||
|
|
||||||
while (vector_isspace(*stringEnd))
|
while (vector_isspace(*stringEnd))
|
||||||
stringEnd++;
|
stringEnd++;
|
||||||
|
|
||||||
|
|||||||
@@ -12,14 +12,11 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
|||||||
[0,0,0]
|
[0,0,0]
|
||||||
(4 rows)
|
(4 rows)
|
||||||
|
|
||||||
SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
|
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
|
||||||
val
|
count
|
||||||
---------
|
-------
|
||||||
[0,0,0]
|
4
|
||||||
[1,1,1]
|
(1 row)
|
||||||
[1,2,3]
|
|
||||||
[1,2,4]
|
|
||||||
(4 rows)
|
|
||||||
|
|
||||||
SELECT COUNT(*) FROM t;
|
SELECT COUNT(*) FROM t;
|
||||||
count
|
count
|
||||||
|
|||||||
@@ -63,9 +63,21 @@ SELECT '[1.5e-38,-1.5e-38]'::vector;
|
|||||||
(1 row)
|
(1 row)
|
||||||
|
|
||||||
SELECT '[4e38,1]'::vector;
|
SELECT '[4e38,1]'::vector;
|
||||||
ERROR: infinite value not allowed in vector
|
ERROR: "4e38" is out of range for type vector
|
||||||
LINE 1: SELECT '[4e38,1]'::vector;
|
LINE 1: SELECT '[4e38,1]'::vector;
|
||||||
^
|
^
|
||||||
|
SELECT '[-4e38,1]'::vector;
|
||||||
|
ERROR: "-4e38" is out of range for type vector
|
||||||
|
LINE 1: SELECT '[-4e38,1]'::vector;
|
||||||
|
^
|
||||||
|
SELECT '[1e-46,1]'::vector;
|
||||||
|
ERROR: "1e-46" is out of range for type vector
|
||||||
|
LINE 1: SELECT '[1e-46,1]'::vector;
|
||||||
|
^
|
||||||
|
SELECT '[-1e-46,1]'::vector;
|
||||||
|
ERROR: "-1e-46" is out of range for type vector
|
||||||
|
LINE 1: SELECT '[-1e-46,1]'::vector;
|
||||||
|
^
|
||||||
SELECT '[1,2,3'::vector;
|
SELECT '[1,2,3'::vector;
|
||||||
ERROR: malformed vector literal: "[1,2,3"
|
ERROR: malformed vector literal: "[1,2,3"
|
||||||
LINE 1: SELECT '[1,2,3'::vector;
|
LINE 1: SELECT '[1,2,3'::vector;
|
||||||
@@ -116,8 +128,30 @@ SELECT '[1, ,3]'::vector;
|
|||||||
ERROR: invalid input syntax for type vector: "[1, ,3]"
|
ERROR: invalid input syntax for type vector: "[1, ,3]"
|
||||||
LINE 1: SELECT '[1, ,3]'::vector;
|
LINE 1: SELECT '[1, ,3]'::vector;
|
||||||
^
|
^
|
||||||
|
SELECT '[1,2,3]'::vector(3);
|
||||||
|
vector
|
||||||
|
---------
|
||||||
|
[1,2,3]
|
||||||
|
(1 row)
|
||||||
|
|
||||||
SELECT '[1,2,3]'::vector(2);
|
SELECT '[1,2,3]'::vector(2);
|
||||||
ERROR: expected 2 dimensions, not 3
|
ERROR: expected 2 dimensions, not 3
|
||||||
|
SELECT '[1,2,3]'::vector(3, 2);
|
||||||
|
ERROR: invalid type modifier
|
||||||
|
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
|
||||||
|
^
|
||||||
|
SELECT '[1,2,3]'::vector('a');
|
||||||
|
ERROR: invalid input syntax for type integer: "a"
|
||||||
|
LINE 1: SELECT '[1,2,3]'::vector('a');
|
||||||
|
^
|
||||||
|
SELECT '[1,2,3]'::vector(0);
|
||||||
|
ERROR: dimensions for type vector must be at least 1
|
||||||
|
LINE 1: SELECT '[1,2,3]'::vector(0);
|
||||||
|
^
|
||||||
|
SELECT '[1,2,3]'::vector(16001);
|
||||||
|
ERROR: dimensions for type vector cannot exceed 16000
|
||||||
|
LINE 1: SELECT '[1,2,3]'::vector(16001);
|
||||||
|
^
|
||||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
||||||
unnest
|
unnest
|
||||||
---------
|
---------
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops);
|
|||||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||||
|
|
||||||
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||||
SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
|
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
|
||||||
SELECT COUNT(*) FROM t;
|
SELECT COUNT(*) FROM t;
|
||||||
|
|
||||||
TRUNCATE t;
|
TRUNCATE t;
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ SELECT '[1.5e38,-1.5e38]'::vector;
|
|||||||
SELECT '[1.5e+38,-1.5e+38]'::vector;
|
SELECT '[1.5e+38,-1.5e+38]'::vector;
|
||||||
SELECT '[1.5e-38,-1.5e-38]'::vector;
|
SELECT '[1.5e-38,-1.5e-38]'::vector;
|
||||||
SELECT '[4e38,1]'::vector;
|
SELECT '[4e38,1]'::vector;
|
||||||
|
SELECT '[-4e38,1]'::vector;
|
||||||
|
SELECT '[1e-46,1]'::vector;
|
||||||
|
SELECT '[-1e-46,1]'::vector;
|
||||||
SELECT '[1,2,3'::vector;
|
SELECT '[1,2,3'::vector;
|
||||||
SELECT '[1,2,3]9'::vector;
|
SELECT '[1,2,3]9'::vector;
|
||||||
SELECT '1,2,3'::vector;
|
SELECT '1,2,3'::vector;
|
||||||
@@ -22,7 +25,13 @@ SELECT '[1,]'::vector;
|
|||||||
SELECT '[1a]'::vector;
|
SELECT '[1a]'::vector;
|
||||||
SELECT '[1,,3]'::vector;
|
SELECT '[1,,3]'::vector;
|
||||||
SELECT '[1, ,3]'::vector;
|
SELECT '[1, ,3]'::vector;
|
||||||
|
|
||||||
|
SELECT '[1,2,3]'::vector(3);
|
||||||
SELECT '[1,2,3]'::vector(2);
|
SELECT '[1,2,3]'::vector(2);
|
||||||
|
SELECT '[1,2,3]'::vector(3, 2);
|
||||||
|
SELECT '[1,2,3]'::vector('a');
|
||||||
|
SELECT '[1,2,3]'::vector(0);
|
||||||
|
SELECT '[1,2,3]'::vector(16001);
|
||||||
|
|
||||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
||||||
SELECT '{"[1,2,3]"}'::vector(2)[];
|
SELECT '{"[1,2,3]"}'::vector(2)[];
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ foreach (@queries)
|
|||||||
push(@expected, $res);
|
push(@expected, $res);
|
||||||
}
|
}
|
||||||
|
|
||||||
test_recall(0.20, $limit, "before vacuum");
|
test_recall(0.19, $limit, "before vacuum");
|
||||||
test_recall(0.95, 100, "before vacuum");
|
test_recall(0.95, 100, "before vacuum");
|
||||||
|
|
||||||
# TODO Test concurrent inserts with vacuum
|
# TODO Test concurrent inserts with vacuum
|
||||||
|
|||||||
Reference in New Issue
Block a user