mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
42 Commits
windows-no
...
hnsw
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0c0cb3e35e | ||
|
|
19c7b4e85b | ||
|
|
a4a6ca6780 | ||
|
|
d027cb586e | ||
|
|
0ffdefe138 | ||
|
|
72a8f68dc5 | ||
|
|
11e0b87abe | ||
|
|
405b7d98dc | ||
|
|
7c217bad0d | ||
|
|
f82c90ce14 | ||
|
|
3390eb783b | ||
|
|
247bc14ca1 | ||
|
|
8f0e865137 | ||
|
|
7194c0281c | ||
|
|
f55bf54588 | ||
|
|
3a6cd544ff | ||
|
|
4f4f444396 | ||
|
|
89835d394e | ||
|
|
348bfb71ff | ||
|
|
f27ac1e338 | ||
|
|
6e0ddf26f0 | ||
|
|
029c336c62 | ||
|
|
7f4acf9d43 | ||
|
|
fe934c1465 | ||
|
|
e8a6becff7 | ||
|
|
62067b298d | ||
|
|
3424b49033 | ||
|
|
c49557674c | ||
|
|
8a89f9deb6 | ||
|
|
1a4fce10be | ||
|
|
18b315bb08 | ||
|
|
fae2b445d6 | ||
|
|
f3bd7c30d4 | ||
|
|
5802009c86 | ||
|
|
3169a60e5c | ||
|
|
e2a8dd6594 | ||
|
|
ae78f732ef | ||
|
|
d3e08fdf99 | ||
|
|
95eded091f | ||
|
|
9e115c629f | ||
|
|
d902e1daff | ||
|
|
f0760eee76 |
22
.github/workflows/build.yml
vendored
22
.github/workflows/build.yml
vendored
@@ -77,5 +77,23 @@ jobs:
|
||||
nmake /NOLOGO /F Makefile.win clean && ^
|
||||
nmake /NOLOGO /F Makefile.win uninstall
|
||||
shell: cmd
|
||||
- if: ${{ failure() }}
|
||||
run: cat regression.diffs
|
||||
i386:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: debian:11
|
||||
options: --platform linux/386
|
||||
steps:
|
||||
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-13 postgresql-server-dev-13 sudo
|
||||
- run: service postgresql start
|
||||
- run: |
|
||||
git clone https://github.com/${{ github.repository }}.git pgvector
|
||||
cd pgvector
|
||||
git fetch origin ${{ github.ref }}
|
||||
git reset --hard FETCH_HEAD
|
||||
make
|
||||
make install
|
||||
chown -R postgres .
|
||||
sudo -u postgres make installcheck
|
||||
sudo -u postgres make prove_installcheck
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
|
||||
@@ -3,12 +3,9 @@
|
||||
- Added HNSW index type
|
||||
- Added support for parallel index builds
|
||||
- Added `l1_distance` function
|
||||
- Added `normalize_l2` function
|
||||
- Added element-wise multiplication for vectors
|
||||
- Added `sum` aggregate
|
||||
- Improved performance of distance functions
|
||||
- Fixed out of range results for cosine distance
|
||||
- Fixed results for NULL and NaN distances
|
||||
|
||||
## 0.4.4 (2023-06-12)
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ ARG PG_MAJOR
|
||||
COPY . /tmp/pgvector
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-mark hold locales && \
|
||||
apt-get install -y --no-install-recommends build-essential postgresql-server-dev-$PG_MAJOR && \
|
||||
cd /tmp/pgvector && \
|
||||
make clean && \
|
||||
@@ -16,5 +15,4 @@ RUN apt-get update && \
|
||||
rm -r /tmp/pgvector && \
|
||||
apt-get remove -y build-essential postgresql-server-dev-$PG_MAJOR && \
|
||||
apt-get autoremove -y && \
|
||||
apt-mark unhold locales && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
84
README.md
84
README.md
@@ -159,6 +159,15 @@ By default, pgvector performs exact nearest neighbor search, which provides perf
|
||||
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for performance. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Supported index types are:
|
||||
|
||||
- [IVFFlat](#ivfflat)
|
||||
- [HNSW](#hnsw) (*coming in 0.5.0*)
|
||||
|
||||
## IVFFlat
|
||||
|
||||
An IVFFlat index clusters vectors into lists, and then searches a subset of those lists. It has faster build times and uses less memory than HNSW, but has lower query performance.
|
||||
|
||||
Three keys to achieving good recall are:
|
||||
|
||||
1. Create the index *after* the table has some data
|
||||
@@ -206,7 +215,57 @@ SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
### Indexing Progress
|
||||
## HNSW
|
||||
|
||||
An HNSW index creates a multilayer graph between vectors. It has slower build times and uses more memory than IVFFlat, but has better query performance. There’s no training step like IVFFlat, so the index can be created without any data in the table.
|
||||
|
||||
The options for HNSW are:
|
||||
|
||||
- `m` - the max number of connections per layer (the bottom layer uses `2 * m`)
|
||||
- `ef_construction` - the size of the dynamic candidate list for constructing the graph
|
||||
|
||||
Add an index for each distance function you want to use.
|
||||
|
||||
L2 distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WITH (m = 16, ef_construction = 40);
|
||||
```
|
||||
|
||||
Inner product
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_ip_ops) WITH (m = 16, ef_construction = 40);
|
||||
```
|
||||
|
||||
Cosine distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 40);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
|
||||
### Query Options
|
||||
|
||||
Specify the size of the dynamic candidate list for search (40 by default)
|
||||
|
||||
```sql
|
||||
SET hnsw.ef_search = 100;
|
||||
```
|
||||
|
||||
A higher value provides better recall at the cost of speed.
|
||||
|
||||
Use `SET LOCAL` inside a transaction to set it for a single query
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
SET LOCAL hnsw.ef_search = 100;
|
||||
SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## Indexing Progress
|
||||
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
|
||||
|
||||
@@ -217,8 +276,8 @@ SELECT phase, tuples_done, tuples_total FROM pg_stat_progress_create_index;
|
||||
The phases are:
|
||||
|
||||
1. `initializing`
|
||||
2. `performing k-means`
|
||||
3. `sorting tuples`
|
||||
2. `performing k-means` (IVFFlat only)
|
||||
3. `sorting tuples` (IVFFlat only)
|
||||
4. `loading tuples`
|
||||
|
||||
Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
|
||||
@@ -283,7 +342,7 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
|
||||
### Approximate Search
|
||||
|
||||
To speed up queries with an index, increase the number of inverted lists (at the expense of recall).
|
||||
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
@@ -298,7 +357,6 @@ Language | Libraries / Examples
|
||||
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
|
||||
C# | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
|
||||
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
|
||||
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
|
||||
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
|
||||
Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
|
||||
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
|
||||
@@ -359,7 +417,7 @@ or choose to store vectors inline:
|
||||
ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
|
||||
```
|
||||
|
||||
#### Why are there less results for a query after adding an index?
|
||||
#### Why are there less results for a query after adding an IVFFlat index?
|
||||
|
||||
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
||||
|
||||
@@ -392,7 +450,6 @@ cosine_distance(vector, vector) → double precision | cosine distance
|
||||
inner_product(vector, vector) → double precision | inner product
|
||||
l2_distance(vector, vector) → double precision | Euclidean distance
|
||||
l1_distance(vector, vector) → double precision | taxicab distance [unreleased]
|
||||
normalize_l2(vector) → vector | normalize with Euclidean norm [unreleased]
|
||||
vector_dims(vector) → integer | number of dimensions
|
||||
vector_norm(vector) → double precision | Euclidean norm
|
||||
|
||||
@@ -433,15 +490,7 @@ Note: Replace `15` with your Postgres server version
|
||||
|
||||
### Windows
|
||||
|
||||
Support for Windows is currently experimental. Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
|
||||
|
||||
```cmd
|
||||
call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
|
||||
```
|
||||
|
||||
Note: The exact path will vary depending on your Visual Studio version and edition
|
||||
|
||||
Then use `nmake` to build:
|
||||
Support for Windows is currently experimental. Use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\15"
|
||||
@@ -468,7 +517,6 @@ You can also build the image manually:
|
||||
```sh
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
git cherry-pick 237a6df
|
||||
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
@@ -570,7 +618,7 @@ Thanks to:
|
||||
|
||||
- [PASE: PostgreSQL Ultra-High-Dimensional Approximate Nearest Neighbor Search Extension](https://dl.acm.org/doi/pdf/10.1145/3318464.3386131)
|
||||
- [Faiss: A Library for Efficient Similarity Search and Clustering of Dense Vectors](https://github.com/facebookresearch/faiss)
|
||||
- [Using the Triangle Inequality to Accelerate k-means](https://cdn.aaai.org/ICML/2003/ICML03-022.pdf)
|
||||
- [Using the Triangle Inequality to Accelerate k-means](https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf)
|
||||
- [k-means++: The Advantage of Careful Seeding](https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf)
|
||||
- [Concept Decompositions for Large Sparse Text Data using Clustering](https://www.cs.utexas.edu/users/inderjit/public_papers/concept_mlj.pdf)
|
||||
- [Efficient and Robust Approximate Nearest Neighbor Search using Hierarchical Navigable Small World Graphs](https://arxiv.org/ftp/arxiv/papers/1603/1603.09320.pdf)
|
||||
|
||||
@@ -4,9 +4,6 @@
|
||||
CREATE FUNCTION l1_distance(vector, vector) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION normalize_l2(vector) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
|
||||
@@ -49,9 +49,6 @@ CREATE FUNCTION vector_dims(vector) RETURNS integer
|
||||
CREATE FUNCTION vector_norm(vector) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION normalize_l2(vector) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION vector_add(vector, vector) RETURNS vector
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
|
||||
17
src/hnsw.h
17
src/hnsw.h
@@ -49,7 +49,7 @@
|
||||
#define PROGRESS_HNSW_PHASE_LOAD 2
|
||||
|
||||
#define HNSW_ELEMENT_TUPLE_SIZE(_dim) MAXALIGN(offsetof(HnswElementTupleData, vec) + VECTOR_SIZE(_dim))
|
||||
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
|
||||
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, neighbors) + ((level) + 2) * (m) * sizeof(HnswNeighborTupleItem))
|
||||
|
||||
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
|
||||
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
|
||||
@@ -71,9 +71,6 @@
|
||||
#define HnswGetLayerM(m, layer) (layer == 0 ? m * 2 : m)
|
||||
#define HnswGetMl(m) (1 / log(m))
|
||||
|
||||
/* Ensure fits in uint8 */
|
||||
#define HnswGetMaxLevel(m) Min(((BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - offsetof(HnswNeighborTupleData, indextids) - sizeof(ItemIdData)) / (sizeof(ItemPointerData)) / m) - 2, 255)
|
||||
|
||||
/* Variables */
|
||||
extern int hnsw_ef_search;
|
||||
|
||||
@@ -200,12 +197,19 @@ typedef struct HnswElementTupleData
|
||||
|
||||
typedef HnswElementTupleData * HnswElementTuple;
|
||||
|
||||
typedef struct HnswNeighborTupleItem
|
||||
{
|
||||
ItemPointerData indextid;
|
||||
uint16 unused;
|
||||
float distance; /* improves performance of inserts */
|
||||
} HnswNeighborTupleItem;
|
||||
|
||||
typedef struct HnswNeighborTupleData
|
||||
{
|
||||
uint8 type;
|
||||
uint8 unused;
|
||||
uint16 count;
|
||||
ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
|
||||
HnswNeighborTupleItem neighbors[FLEXIBLE_ARRAY_MEMBER];
|
||||
} HnswNeighborTupleData;
|
||||
|
||||
typedef HnswNeighborTupleData * HnswNeighborTuple;
|
||||
@@ -291,4 +295,7 @@ void hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys,
|
||||
bool hnswgettuple(IndexScanDesc scan, ScanDirection dir);
|
||||
void hnswendscan(IndexScanDesc scan);
|
||||
|
||||
/* Ensure fits in uint8 */
|
||||
#define HnswGetMaxLevel(m) Min(((BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - offsetof(HnswNeighborTupleData, neighbors) - sizeof(ItemIdData)) / (sizeof(HnswNeighborTupleItem)) / m) - 2, 255)
|
||||
|
||||
#endif
|
||||
|
||||
@@ -317,10 +317,11 @@ UpdateNeighborPages(Relation index, HnswElement e, int m, List *updates)
|
||||
/* Make robust to issues */
|
||||
if (idx < ntup->count)
|
||||
{
|
||||
ItemPointer indextid = &ntup->indextids[idx];
|
||||
HnswNeighborTupleItem *neighbor = &ntup->neighbors[idx];
|
||||
|
||||
/* Update neighbor */
|
||||
ItemPointerSet(indextid, e->blkno, e->offno);
|
||||
ItemPointerSet(&neighbor->indextid, e->blkno, e->offno);
|
||||
neighbor->distance = update->hc.distance;
|
||||
|
||||
/* Overwrite tuple */
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) ntup, ntupSize))
|
||||
|
||||
@@ -309,16 +309,20 @@ HnswSetNeighborTuple(HnswNeighborTuple ntup, HnswElement e, int m)
|
||||
|
||||
for (int i = 0; i < lm; i++)
|
||||
{
|
||||
ItemPointer indextid = &ntup->indextids[idx++];
|
||||
HnswNeighborTupleItem *neighbor = &ntup->neighbors[idx++];
|
||||
|
||||
if (i < neighbors->length)
|
||||
{
|
||||
HnswCandidate *hc = &neighbors->items[i];
|
||||
|
||||
ItemPointerSet(indextid, hc->element->blkno, hc->element->offno);
|
||||
ItemPointerSet(&neighbor->indextid, hc->element->blkno, hc->element->offno);
|
||||
neighbor->distance = hc->distance;
|
||||
}
|
||||
else
|
||||
ItemPointerSetInvalid(indextid);
|
||||
{
|
||||
ItemPointerSetInvalid(&neighbor->indextid);
|
||||
neighbor->distance = NAN;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,15 +352,15 @@ LoadNeighborsFromPage(HnswElement element, Relation index, Page page)
|
||||
HnswElement e;
|
||||
int level;
|
||||
HnswCandidate *hc;
|
||||
ItemPointer indextid;
|
||||
HnswNeighborTupleItem *neighbor;
|
||||
HnswNeighborArray *neighbors;
|
||||
|
||||
indextid = &ntup->indextids[i];
|
||||
neighbor = &ntup->neighbors[i];
|
||||
|
||||
if (!ItemPointerIsValid(indextid))
|
||||
if (!ItemPointerIsValid(&neighbor->indextid))
|
||||
continue;
|
||||
|
||||
e = InitElementFromBlock(ItemPointerGetBlockNumber(indextid), ItemPointerGetOffsetNumber(indextid));
|
||||
e = InitElementFromBlock(ItemPointerGetBlockNumber(&neighbor->indextid), ItemPointerGetOffsetNumber(&neighbor->indextid));
|
||||
|
||||
/* Calculate level based on offset */
|
||||
level = element->level - i / m;
|
||||
@@ -366,6 +370,7 @@ LoadNeighborsFromPage(HnswElement element, Relation index, Page page)
|
||||
neighbors = &element->neighbors[level];
|
||||
hc = &neighbors->items[neighbors->length++];
|
||||
hc->element = e;
|
||||
hc->distance = neighbor->distance;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -845,37 +850,33 @@ UpdateConnections(HnswElement element, List *neighbors, int m, int lc, List **up
|
||||
HnswCandidate *pruned = NULL;
|
||||
List *c = NIL;
|
||||
|
||||
/* Add and sort candidates */
|
||||
for (int i = 0; i < currentNeighbors->length; i++)
|
||||
c = lappend(c, ¤tNeighbors->items[i]);
|
||||
c = lappend(c, &hc2);
|
||||
list_sort(c, CompareCandidateDistances);
|
||||
|
||||
/* Load elements on insert */
|
||||
if (index != NULL)
|
||||
{
|
||||
Datum q = PointerGetDatum(hc->element->vec);
|
||||
|
||||
for (int i = 0; i < currentNeighbors->length; i++)
|
||||
{
|
||||
HnswCandidate *hc3 = ¤tNeighbors->items[i];
|
||||
|
||||
if (hc3->element->vec == NULL)
|
||||
HnswLoadElement(hc3->element, &hc3->distance, &q, index, procinfo, collation, true);
|
||||
else
|
||||
hc3->distance = GetCandidateDistance(hc3, q, procinfo, collation);
|
||||
|
||||
/* Prune element if being deleted */
|
||||
if (list_length(hc3->element->heaptids) == 0)
|
||||
if (currentNeighbors->items[i].element->vec == NULL)
|
||||
{
|
||||
pruned = ¤tNeighbors->items[i];
|
||||
break;
|
||||
HnswLoadElement(currentNeighbors->items[i].element, NULL, NULL, index, procinfo, collation, true);
|
||||
|
||||
/* Prune deleted element */
|
||||
if (currentNeighbors->items[i].element->deleted)
|
||||
{
|
||||
pruned = ¤tNeighbors->items[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pruned == NULL)
|
||||
{
|
||||
/* Add and sort candidates */
|
||||
for (int i = 0; i < currentNeighbors->length; i++)
|
||||
c = lappend(c, ¤tNeighbors->items[i]);
|
||||
c = lappend(c, &hc2);
|
||||
list_sort(c, CompareCandidateDistances);
|
||||
|
||||
SelectNeighbors(c, m, lc, procinfo, collation, &pruned);
|
||||
|
||||
/* Should not happen */
|
||||
|
||||
@@ -156,13 +156,13 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
|
||||
/* Check neighbors */
|
||||
for (int i = 0; i < ntup->count; i++)
|
||||
{
|
||||
ItemPointer indextid = &ntup->indextids[i];
|
||||
HnswNeighborTupleItem *neighbor = &ntup->neighbors[i];
|
||||
|
||||
if (!ItemPointerIsValid(indextid))
|
||||
if (!ItemPointerIsValid(&neighbor->indextid))
|
||||
continue;
|
||||
|
||||
/* Check if in deleted list */
|
||||
if (DeletedContains(vacuumstate->deleted, indextid))
|
||||
if (DeletedContains(vacuumstate->deleted, &neighbor->indextid))
|
||||
{
|
||||
needsUpdated = true;
|
||||
break;
|
||||
@@ -453,7 +453,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
|
||||
/* Overwrite neighbors */
|
||||
for (int i = 0; i < ntup->count; i++)
|
||||
ItemPointerSetInvalid(&ntup->indextids[i]);
|
||||
{
|
||||
ItemPointerSetInvalid(&ntup->neighbors[i].indextid);
|
||||
ntup->neighbors[i].distance = NAN;
|
||||
}
|
||||
|
||||
/* Overwrite element tuple */
|
||||
if (!PageIndexTupleOverwrite(page, offno, (Item) etup, etupSize))
|
||||
|
||||
@@ -180,29 +180,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
tuplesort_performsort(so->sortstate);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get dimensions from metapage
|
||||
*/
|
||||
static int
|
||||
GetDimensions(Relation index)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
IvfflatMetaPage metap;
|
||||
int dimensions;
|
||||
|
||||
buf = ReadBuffer(index, IVFFLAT_METAPAGE_BLKNO);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
metap = IvfflatPageGetMeta(page);
|
||||
|
||||
dimensions = metap->dimensions;
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
/*
|
||||
* Prepare for an index scan
|
||||
*/
|
||||
@@ -308,19 +285,21 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
if (scan->orderByData == NULL)
|
||||
elog(ERROR, "cannot scan ivfflat index without order");
|
||||
|
||||
/* No items will match if null */
|
||||
if (scan->orderByData->sk_flags & SK_ISNULL)
|
||||
value = PointerGetDatum(InitVector(GetDimensions(scan->indexRelation)));
|
||||
else
|
||||
return false;
|
||||
|
||||
value = scan->orderByData->sk_argument;
|
||||
|
||||
/* Value should not be compressed or toasted */
|
||||
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
|
||||
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
|
||||
|
||||
if (so->normprocinfo != NULL)
|
||||
{
|
||||
value = scan->orderByData->sk_argument;
|
||||
|
||||
/* Value should not be compressed or toasted */
|
||||
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
|
||||
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
|
||||
|
||||
/* Fine if normalization fails */
|
||||
if (so->normprocinfo != NULL)
|
||||
IvfflatNormValue(so->normprocinfo, so->collation, &value, NULL);
|
||||
/* No items will match if normalization fails */
|
||||
if (!IvfflatNormValue(so->normprocinfo, so->collation, &value, NULL))
|
||||
return false;
|
||||
}
|
||||
|
||||
IvfflatBench("GetScanLists", GetScanLists(scan, value));
|
||||
|
||||
57
src/vector.c
57
src/vector.c
@@ -632,7 +632,6 @@ cosine_distance(PG_FUNCTION_ARGS)
|
||||
float distance = 0.0;
|
||||
float norma = 0.0;
|
||||
float normb = 0.0;
|
||||
double similarity;
|
||||
|
||||
CheckDims(a, b);
|
||||
|
||||
@@ -645,21 +644,7 @@ cosine_distance(PG_FUNCTION_ARGS)
|
||||
}
|
||||
|
||||
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
|
||||
similarity = (double) distance / sqrt((double) norma * (double) normb);
|
||||
|
||||
#ifdef _MSC_VER
|
||||
/* /fp:fast may not propagate NaN */
|
||||
if (isnan(similarity))
|
||||
PG_RETURN_FLOAT8(NAN);
|
||||
#endif
|
||||
|
||||
/* Keep in range */
|
||||
if (similarity > 1)
|
||||
similarity = 1.0;
|
||||
else if (similarity < -1)
|
||||
similarity = -1.0;
|
||||
|
||||
PG_RETURN_FLOAT8(1.0 - similarity);
|
||||
PG_RETURN_FLOAT8(1.0 - ((double) distance / sqrt((double) norma * (double) normb)));
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -683,7 +668,6 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
dp += a->x[i] * b->x[i];
|
||||
|
||||
distance = (double) dp;
|
||||
|
||||
/* Prevent NaN with acos with loss of precision */
|
||||
if (distance > 1)
|
||||
distance = 1;
|
||||
@@ -745,45 +729,6 @@ vector_norm(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_FLOAT8(sqrt((double) norm));
|
||||
}
|
||||
|
||||
/*
|
||||
* Normalize a vector with the L2 norm
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(normalize_l2);
|
||||
Datum
|
||||
normalize_l2(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
float *ax = a->x;
|
||||
double norm = 0.0;
|
||||
Vector *result;
|
||||
float *rx;
|
||||
|
||||
result = InitVector(a->dim);
|
||||
rx = result->x;
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
norm += (double) ax[i] * (double) ax[i];
|
||||
|
||||
norm = sqrt(norm);
|
||||
|
||||
if (norm > 0)
|
||||
{
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] / norm;
|
||||
|
||||
/* Check for overflow */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
{
|
||||
if (isinf(rx[i]))
|
||||
float_overflow_error();
|
||||
}
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Add vectors
|
||||
*/
|
||||
|
||||
@@ -48,36 +48,6 @@ SELECT vector_norm('[0,1]');
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT normalize_l2('[3,4]');
|
||||
normalize_l2
|
||||
--------------
|
||||
[0.6,0.8]
|
||||
(1 row)
|
||||
|
||||
SELECT normalize_l2('[3,0]');
|
||||
normalize_l2
|
||||
--------------
|
||||
[1,0]
|
||||
(1 row)
|
||||
|
||||
SELECT normalize_l2('[0,0.1]');
|
||||
normalize_l2
|
||||
--------------
|
||||
[0,1]
|
||||
(1 row)
|
||||
|
||||
SELECT normalize_l2('[0,0]');
|
||||
normalize_l2
|
||||
--------------
|
||||
[0,0]
|
||||
(1 row)
|
||||
|
||||
SELECT normalize_l2('[3e38]');
|
||||
normalize_l2
|
||||
--------------
|
||||
[1]
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[0,0]', '[3,4]');
|
||||
l2_distance
|
||||
-------------
|
||||
@@ -92,12 +62,6 @@ SELECT l2_distance('[0,0]', '[0,1]');
|
||||
|
||||
SELECT l2_distance('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT l2_distance('[3e38]', '[-3e38]');
|
||||
l2_distance
|
||||
-------------
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]', '[3,4]');
|
||||
inner_product
|
||||
---------------
|
||||
@@ -106,12 +70,6 @@ SELECT inner_product('[1,2]', '[3,4]');
|
||||
|
||||
SELECT inner_product('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT inner_product('[3e38]', '[3e38]');
|
||||
inner_product
|
||||
---------------
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]', '[2,4]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
@@ -138,24 +96,6 @@ SELECT cosine_distance('[1,1]', '[-1,-1]');
|
||||
|
||||
SELECT cosine_distance('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[3e38]', '[3e38]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT l1_distance('[0,0]', '[3,4]');
|
||||
l1_distance
|
||||
-------------
|
||||
@@ -170,12 +110,6 @@ SELECT l1_distance('[0,0]', '[0,1]');
|
||||
|
||||
SELECT l1_distance('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT l1_distance('[3e38]', '[-3e38]');
|
||||
l1_distance
|
||||
-------------
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
avg
|
||||
-----------
|
||||
|
||||
@@ -11,16 +11,9 @@ SELECT * FROM t ORDER BY val <=> '[3,3,3]';
|
||||
[1,2,4]
|
||||
(3 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
|
||||
count
|
||||
-------
|
||||
3
|
||||
(1 row)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
|
||||
count
|
||||
-------
|
||||
3
|
||||
(1 row)
|
||||
SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector);
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -12,10 +12,9 @@ SELECT * FROM t ORDER BY val <#> '[3,3,3]';
|
||||
[0,0,0]
|
||||
(4 rows)
|
||||
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
|
||||
count
|
||||
-------
|
||||
4
|
||||
(1 row)
|
||||
SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector);
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -13,13 +13,9 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]';
|
||||
(4 rows)
|
||||
|
||||
SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
|
||||
val
|
||||
---------
|
||||
[0,0,0]
|
||||
[1,1,1]
|
||||
[1,2,3]
|
||||
[1,2,4]
|
||||
(4 rows)
|
||||
val
|
||||
-----
|
||||
(0 rows)
|
||||
|
||||
SELECT COUNT(*) FROM t;
|
||||
count
|
||||
|
||||
@@ -12,34 +12,22 @@ SELECT round(vector_norm('[1,1]')::numeric, 5);
|
||||
SELECT vector_norm('[3,4]');
|
||||
SELECT vector_norm('[0,1]');
|
||||
|
||||
SELECT normalize_l2('[3,4]');
|
||||
SELECT normalize_l2('[3,0]');
|
||||
SELECT normalize_l2('[0,0.1]');
|
||||
SELECT normalize_l2('[0,0]');
|
||||
SELECT normalize_l2('[3e38]');
|
||||
|
||||
SELECT l2_distance('[0,0]', '[3,4]');
|
||||
SELECT l2_distance('[0,0]', '[0,1]');
|
||||
SELECT l2_distance('[1,2]', '[3]');
|
||||
SELECT l2_distance('[3e38]', '[-3e38]');
|
||||
|
||||
SELECT inner_product('[1,2]', '[3,4]');
|
||||
SELECT inner_product('[1,2]', '[3]');
|
||||
SELECT inner_product('[3e38]', '[3e38]');
|
||||
|
||||
SELECT cosine_distance('[1,2]', '[2,4]');
|
||||
SELECT cosine_distance('[1,2]', '[0,0]');
|
||||
SELECT cosine_distance('[1,1]', '[1,1]');
|
||||
SELECT cosine_distance('[1,1]', '[-1,-1]');
|
||||
SELECT cosine_distance('[1,2]', '[3]');
|
||||
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
|
||||
SELECT cosine_distance('[3e38]', '[3e38]');
|
||||
|
||||
SELECT l1_distance('[0,0]', '[3,4]');
|
||||
SELECT l1_distance('[0,0]', '[0,1]');
|
||||
SELECT l1_distance('[1,2]', '[3]');
|
||||
SELECT l1_distance('[3e38]', '[-3e38]');
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
||||
|
||||
@@ -7,7 +7,6 @@ CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
|
||||
SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector);
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
@@ -7,6 +7,6 @@ CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
|
||||
INSERT INTO t (val) VALUES ('[1,2,4]');
|
||||
|
||||
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
|
||||
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
|
||||
SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector);
|
||||
|
||||
DROP TABLE t;
|
||||
|
||||
Reference in New Issue
Block a user