mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 03:57:34 +08:00
Compare commits
1 Commits
tinyint
...
hnsw-print
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d866844d3 |
11
CHANGELOG.md
11
CHANGELOG.md
@@ -1,18 +1,13 @@
|
||||
## 0.5.1 (unreleased)
|
||||
|
||||
- Added check for MVCC-compliant snapshot for HNSW index scans
|
||||
- Improved performance of index scans for IVFFlat after updates and deletes
|
||||
|
||||
## 0.5.0 (2023-08-28)
|
||||
## 0.5.0 (unreleased)
|
||||
|
||||
- Added HNSW index type
|
||||
- Added support for parallel index builds for IVFFlat
|
||||
- Added support for parallel index builds
|
||||
- Added `l1_distance` function
|
||||
- Added element-wise multiplication for vectors
|
||||
- Added `sum` aggregate
|
||||
- Improved performance of distance functions
|
||||
- Fixed out of range results for cosine distance
|
||||
- Fixed results for NULL and NaN distances for IVFFlat
|
||||
- Fixed results for NULL and NaN distances
|
||||
|
||||
## 0.4.4 (2023-06-12)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.5.0",
|
||||
"version": "0.4.4",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.5.0",
|
||||
"version": "0.4.4",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
8
Makefile
8
Makefile
@@ -1,14 +1,14 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.5.0
|
||||
EXTVERSION = 0.4.4
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
OBJS = src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/tinyint.o src/vector.o
|
||||
HEADERS = src/tinyint.h src/vector.h
|
||||
OBJS = src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/vector.o
|
||||
HEADERS = src/vector.h
|
||||
|
||||
TESTS = $(wildcard test/sql/*.sql)
|
||||
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
|
||||
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
|
||||
REGRESS_OPTS = --inputdir=test --load-extension=vector
|
||||
|
||||
OPTFLAGS = -march=native
|
||||
|
||||
|
||||
10
Makefile.win
10
Makefile.win
@@ -1,11 +1,11 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.5.0
|
||||
EXTVERSION = 0.4.4
|
||||
|
||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\tinyint.obj src\vector.obj
|
||||
HEADERS = src\tinyint.h src\vector.h
|
||||
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||
HEADERS = src\vector.h
|
||||
|
||||
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
|
||||
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
|
||||
REGRESS_OPTS = --inputdir=test --load-extension=vector
|
||||
|
||||
# For /arch flags
|
||||
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-minimum-cpu-architecture
|
||||
@@ -56,7 +56,7 @@ install:
|
||||
copy $(EXTENSION).control "$(SHAREDIR)\extension"
|
||||
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
|
||||
mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
|
||||
for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
|
||||
copy $(HEADERS) "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
|
||||
|
||||
installcheck:
|
||||
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
|
||||
|
||||
132
README.md
132
README.md
@@ -18,7 +18,7 @@ Compile and install the extension (supports Postgres 11+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -157,16 +157,7 @@ SELECT category_id, AVG(embedding) FROM items GROUP BY category_id;
|
||||
|
||||
By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
|
||||
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for speed. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Supported index types are:
|
||||
|
||||
- [IVFFlat](#ivfflat)
|
||||
- [HNSW](#hnsw) - added in 0.5.0
|
||||
|
||||
## IVFFlat
|
||||
|
||||
An IVFFlat index divides vectors into lists, and then searches a subset of those lists that are closest to the query vector. It has faster build times and uses less memory than HNSW, but has lower query performance (in terms of speed-recall tradeoff).
|
||||
You can add an index to use approximate nearest neighbor search, which trades some recall for performance. Unlike typical indexes, you will see different results for queries after adding an approximate index.
|
||||
|
||||
Three keys to achieving good recall are:
|
||||
|
||||
@@ -215,63 +206,7 @@ SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## HNSW
|
||||
|
||||
An HNSW index creates a multilayer graph. It has slower build times and uses more memory than IVFFlat, but has better query performance (in terms of speed-recall tradeoff). There’s no training step like IVFFlat, so the index can be created without any data in the table.
|
||||
|
||||
Add an index for each distance function you want to use.
|
||||
|
||||
L2 distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
|
||||
```
|
||||
|
||||
Inner product
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_ip_ops);
|
||||
```
|
||||
|
||||
Cosine distance
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
|
||||
```
|
||||
|
||||
Vectors with up to 2,000 dimensions can be indexed.
|
||||
|
||||
### Index Options
|
||||
|
||||
Specify HNSW parameters
|
||||
|
||||
- `m` - the max number of connections per layer (16 by default)
|
||||
- `ef_construction` - the size of the dynamic candidate list for constructing the graph (64 by default)
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WITH (m = 16, ef_construction = 64);
|
||||
```
|
||||
|
||||
### Query Options
|
||||
|
||||
Specify the size of the dynamic candidate list for search (40 by default)
|
||||
|
||||
```sql
|
||||
SET hnsw.ef_search = 100;
|
||||
```
|
||||
|
||||
A higher value provides better recall at the cost of speed.
|
||||
|
||||
Use `SET LOCAL` inside a transaction to set it for a single query
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
SET LOCAL hnsw.ef_search = 100;
|
||||
SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## Indexing Progress
|
||||
### Indexing Progress
|
||||
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
|
||||
|
||||
@@ -282,8 +217,8 @@ SELECT phase, tuples_done, tuples_total FROM pg_stat_progress_create_index;
|
||||
The phases are:
|
||||
|
||||
1. `initializing`
|
||||
2. `performing k-means` - IVFFlat only
|
||||
3. `assigning tuples` - IVFFlat only
|
||||
2. `performing k-means`
|
||||
3. `sorting tuples`
|
||||
4. `loading tuples`
|
||||
|
||||
Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
|
||||
@@ -348,7 +283,7 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
|
||||
### Approximate Search
|
||||
|
||||
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
|
||||
To speed up queries with an index, increase the number of inverted lists (at the expense of recall).
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000);
|
||||
@@ -424,7 +359,7 @@ or choose to store vectors inline:
|
||||
ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
|
||||
```
|
||||
|
||||
#### Why are there less results for a query after adding an IVFFlat index?
|
||||
#### Why are there less results for a query after adding an index?
|
||||
|
||||
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
|
||||
|
||||
@@ -440,32 +375,32 @@ Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a singl
|
||||
|
||||
### Vector Operators
|
||||
|
||||
Operator | Description | Added
|
||||
--- | --- | ---
|
||||
\+ | element-wise addition |
|
||||
\- | element-wise subtraction |
|
||||
\* | element-wise multiplication | 0.5.0
|
||||
<-> | Euclidean distance |
|
||||
<#> | negative inner product |
|
||||
<=> | cosine distance |
|
||||
Operator | Description
|
||||
--- | ---
|
||||
\+ | element-wise addition
|
||||
\- | element-wise subtraction
|
||||
\* | element-wise multiplication [unreleased]
|
||||
<-> | Euclidean distance
|
||||
<#> | negative inner product
|
||||
<=> | cosine distance
|
||||
|
||||
### Vector Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
cosine_distance(vector, vector) → double precision | cosine distance |
|
||||
inner_product(vector, vector) → double precision | inner product |
|
||||
l2_distance(vector, vector) → double precision | Euclidean distance |
|
||||
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
|
||||
vector_dims(vector) → integer | number of dimensions |
|
||||
vector_norm(vector) → double precision | Euclidean norm |
|
||||
Function | Description
|
||||
--- | ---
|
||||
cosine_distance(vector, vector) → double precision | cosine distance
|
||||
inner_product(vector, vector) → double precision | inner product
|
||||
l2_distance(vector, vector) → double precision | Euclidean distance
|
||||
l1_distance(vector, vector) → double precision | taxicab distance [unreleased]
|
||||
vector_dims(vector) → integer | number of dimensions
|
||||
vector_norm(vector) → double precision | Euclidean norm
|
||||
|
||||
### Aggregate Functions
|
||||
|
||||
Function | Description | Added
|
||||
--- | --- | ---
|
||||
avg(vector) → vector | average |
|
||||
sum(vector) → vector | sum | 0.5.0
|
||||
Function | Description
|
||||
--- | ---
|
||||
avg(vector) → vector | arithmetic mean
|
||||
sum(vector) → vector | sum [unreleased]
|
||||
|
||||
## Installation Notes
|
||||
|
||||
@@ -509,7 +444,7 @@ Then use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\15"
|
||||
git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
@@ -530,8 +465,9 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
git cherry-pick 237a6df
|
||||
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
@@ -595,18 +531,12 @@ pgvector is available on [these providers](https://github.com/pgvector/pgvector/
|
||||
|
||||
## Upgrading
|
||||
|
||||
Install the latest version. Then in each database you want to upgrade, run:
|
||||
Install the latest version and run:
|
||||
|
||||
```sql
|
||||
ALTER EXTENSION vector UPDATE;
|
||||
```
|
||||
|
||||
You can check the version in the current database with:
|
||||
|
||||
```sql
|
||||
SELECT extversion FROM pg_extension WHERE extname = 'vector';
|
||||
```
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
### 0.4.0
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.5.1'" to load this file. \quit
|
||||
|
||||
-- tinyint
|
||||
|
||||
CREATE TYPE tinyint;
|
||||
|
||||
CREATE FUNCTION tinyint_in(cstring, oid, integer) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_out(tinyint) RETURNS cstring
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_recv(internal, oid, integer) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_send(tinyint) RETURNS bytea
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE TYPE tinyint (
|
||||
INPUT = tinyint_in,
|
||||
OUTPUT = tinyint_out,
|
||||
RECEIVE = tinyint_recv,
|
||||
SEND = tinyint_send,
|
||||
INTERNALLENGTH = 1,
|
||||
PASSEDBYVALUE,
|
||||
ALIGNMENT = char
|
||||
);
|
||||
|
||||
CREATE FUNCTION integer_to_tinyint(integer, integer, boolean) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION numeric_to_tinyint(numeric, integer, boolean) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE CAST (integer AS tinyint)
|
||||
WITH FUNCTION integer_to_tinyint(integer, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (numeric AS tinyint)
|
||||
WITH FUNCTION numeric_to_tinyint(numeric, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE FUNCTION l2_distance(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'tinyint_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION inner_product(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'tinyint_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION cosine_distance(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'tinyint_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_negative_inner_product(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = tinyint[], RIGHTARG = tinyint[], PROCEDURE = l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <#> (
|
||||
LEFTARG = tinyint[], RIGHTARG = tinyint[], PROCEDURE = tinyint_negative_inner_product,
|
||||
COMMUTATOR = '<#>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <=> (
|
||||
LEFTARG = tinyint[], RIGHTARG = tinyint[], PROCEDURE = cosine_distance,
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
@@ -290,68 +290,3 @@ CREATE OPERATOR CLASS vector_cosine_ops
|
||||
OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops,
|
||||
FUNCTION 1 vector_negative_inner_product(vector, vector),
|
||||
FUNCTION 2 vector_norm(vector);
|
||||
|
||||
-- tinyint
|
||||
|
||||
CREATE TYPE tinyint;
|
||||
|
||||
CREATE FUNCTION tinyint_in(cstring, oid, integer) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_out(tinyint) RETURNS cstring
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_recv(internal, oid, integer) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_send(tinyint) RETURNS bytea
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE TYPE tinyint (
|
||||
INPUT = tinyint_in,
|
||||
OUTPUT = tinyint_out,
|
||||
RECEIVE = tinyint_recv,
|
||||
SEND = tinyint_send,
|
||||
INTERNALLENGTH = 1,
|
||||
PASSEDBYVALUE,
|
||||
ALIGNMENT = char
|
||||
);
|
||||
|
||||
CREATE FUNCTION integer_to_tinyint(integer, integer, boolean) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION numeric_to_tinyint(numeric, integer, boolean) RETURNS tinyint
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE CAST (integer AS tinyint)
|
||||
WITH FUNCTION integer_to_tinyint(integer, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE CAST (numeric AS tinyint)
|
||||
WITH FUNCTION numeric_to_tinyint(numeric, integer, boolean) AS IMPLICIT;
|
||||
|
||||
CREATE FUNCTION l2_distance(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'tinyint_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION inner_product(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'tinyint_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION cosine_distance(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME', 'tinyint_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE FUNCTION tinyint_negative_inner_product(tinyint[], tinyint[]) RETURNS float8
|
||||
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
|
||||
|
||||
CREATE OPERATOR <-> (
|
||||
LEFTARG = tinyint[], RIGHTARG = tinyint[], PROCEDURE = l2_distance,
|
||||
COMMUTATOR = '<->'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <#> (
|
||||
LEFTARG = tinyint[], RIGHTARG = tinyint[], PROCEDURE = tinyint_negative_inner_product,
|
||||
COMMUTATOR = '<#>'
|
||||
);
|
||||
|
||||
CREATE OPERATOR <=> (
|
||||
LEFTARG = tinyint[], RIGHTARG = tinyint[], PROCEDURE = cosine_distance,
|
||||
COMMUTATOR = '<=>'
|
||||
);
|
||||
|
||||
@@ -91,7 +91,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
MemSet(&costs, 0, sizeof(costs));
|
||||
|
||||
index = index_open(path->indexinfo->indexoid, NoLock);
|
||||
HnswGetMetaPageInfo(index, &m, NULL);
|
||||
m = HnswGetM(index);
|
||||
index_close(index, NoLock);
|
||||
|
||||
/* Approximate entry level */
|
||||
@@ -196,7 +196,7 @@ hnswhandler(PG_FUNCTION_ARGS)
|
||||
amroutine->aminsert = hnswinsert;
|
||||
amroutine->ambulkdelete = hnswbulkdelete;
|
||||
amroutine->amvacuumcleanup = hnswvacuumcleanup;
|
||||
amroutine->amcanreturn = NULL;
|
||||
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
|
||||
amroutine->amcostestimate = hnswcostestimate;
|
||||
amroutine->amoptions = hnswoptions;
|
||||
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
|
||||
|
||||
12
src/hnsw.h
12
src/hnsw.h
@@ -36,7 +36,7 @@
|
||||
#define HNSW_DEFAULT_M 16
|
||||
#define HNSW_MIN_M 2
|
||||
#define HNSW_MAX_M 100
|
||||
#define HNSW_DEFAULT_EF_CONSTRUCTION 64
|
||||
#define HNSW_DEFAULT_EF_CONSTRUCTION 40
|
||||
#define HNSW_MIN_EF_CONSTRUCTION 4
|
||||
#define HNSW_MAX_EF_CONSTRUCTION 1000
|
||||
#define HNSW_DEFAULT_EF_SEARCH 40
|
||||
@@ -57,8 +57,6 @@
|
||||
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
|
||||
#define PROGRESS_HNSW_PHASE_LOAD 2
|
||||
|
||||
#define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData))
|
||||
|
||||
#define HNSW_ELEMENT_TUPLE_SIZE(_dim) MAXALIGN(offsetof(HnswElementTupleData, vec) + VECTOR_SIZE(_dim))
|
||||
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
|
||||
|
||||
@@ -220,6 +218,7 @@ typedef HnswNeighborTupleData * HnswNeighborTuple;
|
||||
typedef struct HnswScanOpaqueData
|
||||
{
|
||||
bool first;
|
||||
Buffer buf;
|
||||
List *w;
|
||||
MemoryContext tmpCtx;
|
||||
|
||||
@@ -260,16 +259,15 @@ typedef struct HnswVacuumState
|
||||
/* Methods */
|
||||
int HnswGetM(Relation index);
|
||||
int HnswGetEfConstruction(Relation index);
|
||||
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
|
||||
FmgrInfo *HnswOptionalProcInfo(Relation rel, uint16 procnum);
|
||||
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
|
||||
void HnswCommitBuffer(Buffer buf, GenericXLogState *state);
|
||||
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
||||
void HnswInitPage(Buffer buf, Page page);
|
||||
void HnswInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state);
|
||||
void HnswInit(void);
|
||||
List *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
|
||||
List *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, bool inserting, HnswElement skipElement);
|
||||
HnswElement HnswGetEntryPoint(Relation index);
|
||||
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
|
||||
HnswElement HnswInitElement(ItemPointer tid, int m, double ml, int maxLevel);
|
||||
void HnswFreeElement(HnswElement element);
|
||||
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
|
||||
@@ -286,7 +284,7 @@ void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool
|
||||
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
|
||||
void HnswSetElementTuple(HnswElementTuple etup, HnswElement element);
|
||||
void HnswUpdateConnection(HnswElement element, HnswCandidate * hc, int m, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
|
||||
void HnswLoadNeighbors(HnswElement element, Relation index, int m);
|
||||
void HnswLoadNeighbors(HnswElement element, Relation index);
|
||||
|
||||
/* Index access methods */
|
||||
IndexBuildResult *hnswbuild(Relation heap, Relation index, IndexInfo *indexInfo);
|
||||
|
||||
@@ -81,6 +81,7 @@ HnswBuildAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **
|
||||
HnswPageGetOpaque(*page)->nextblkno = BufferGetBlockNumber(newbuf);
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(*buf);
|
||||
GenericXLogFinish(*state);
|
||||
UnlockReleaseBuffer(*buf);
|
||||
|
||||
@@ -117,12 +118,12 @@ CreateElementPages(HnswBuildState * buildstate)
|
||||
ListCell *lc;
|
||||
|
||||
/* Calculate sizes */
|
||||
maxSize = HNSW_MAX_SIZE;
|
||||
maxSize = BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData));
|
||||
etupSize = HNSW_ELEMENT_TUPLE_SIZE(dimensions);
|
||||
|
||||
/* Allocate once */
|
||||
etup = palloc0(etupSize);
|
||||
ntup = palloc0(BLCKSZ);
|
||||
ntup = palloc0(maxSize);
|
||||
|
||||
/* Prepare first page */
|
||||
buf = HnswNewBuffer(index, forkNum);
|
||||
@@ -178,6 +179,7 @@ CreateElementPages(HnswBuildState * buildstate)
|
||||
insertPage = BufferGetBlockNumber(buf);
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
@@ -225,6 +227,7 @@ CreateNeighborPages(HnswBuildState * buildstate)
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
|
||||
etupSize = HNSW_ELEMENT_TUPLE_SIZE(dimensions);
|
||||
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m);
|
||||
combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
|
||||
maxSize = HNSW_MAX_SIZE;
|
||||
maxSize = BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData));
|
||||
minCombinedSize = etupSize + HNSW_NEIGHBOR_TUPLE_SIZE(0, m) + sizeof(ItemIdData);
|
||||
|
||||
/* Prepare element tuple */
|
||||
@@ -202,6 +202,8 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
|
||||
HnswInsertAppendPage(index, &newbuf, &newpage, state, page);
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(newbuf);
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
|
||||
/* Unlock previous buffer */
|
||||
@@ -268,6 +270,9 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
|
||||
}
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
if (nbuf != buf)
|
||||
MarkBufferDirty(nbuf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
if (nbuf != buf)
|
||||
@@ -324,7 +329,7 @@ HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswE
|
||||
|
||||
/* Get latest neighbors since they may have changed */
|
||||
/* Do not lock yet since selecting neighbors can take time */
|
||||
HnswLoadNeighbors(hc->element, index, m);
|
||||
HnswLoadNeighbors(hc->element, index);
|
||||
|
||||
/*
|
||||
* Could improve performance for vacuuming by checking neighbors
|
||||
@@ -386,6 +391,7 @@ HnswUpdateNeighborPages(Relation index, FmgrInfo *procinfo, Oid collation, HnswE
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
}
|
||||
else
|
||||
@@ -439,6 +445,7 @@ HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup)
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
@@ -485,8 +492,9 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
|
||||
FmgrInfo *normprocinfo;
|
||||
HnswElement entryPoint;
|
||||
HnswElement element;
|
||||
int m;
|
||||
int m = HnswGetM(index);
|
||||
int efConstruction = HnswGetEfConstruction(index);
|
||||
double ml = HnswGetMl(m);
|
||||
FmgrInfo *procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
|
||||
Oid collation = index->rd_indcollation[0];
|
||||
HnswElement dup;
|
||||
@@ -503,6 +511,10 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Create an element */
|
||||
element = HnswInitElement(heap_tid, m, ml, HnswGetMaxLevel(m));
|
||||
element->vec = DatumGetVector(value);
|
||||
|
||||
/*
|
||||
* Get a shared lock. This allows vacuum to ensure no in-flight inserts
|
||||
* before repairing graph. Use a page lock so it does not interfere with
|
||||
@@ -510,12 +522,8 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
|
||||
*/
|
||||
LockPage(index, HNSW_UPDATE_LOCK, lockmode);
|
||||
|
||||
/* Get m and entry point */
|
||||
HnswGetMetaPageInfo(index, &m, &entryPoint);
|
||||
|
||||
/* Create an element */
|
||||
element = HnswInitElement(heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m));
|
||||
element->vec = DatumGetVector(value);
|
||||
/* Get entry point */
|
||||
entryPoint = HnswGetEntryPoint(index);
|
||||
|
||||
/* Prevent concurrent inserts when likely updating entry point */
|
||||
if (entryPoint == NULL || element->level > entryPoint->level)
|
||||
|
||||
@@ -19,11 +19,7 @@ GetScanItems(IndexScanDesc scan, Datum q)
|
||||
Oid collation = so->collation;
|
||||
List *ep;
|
||||
List *w;
|
||||
int m;
|
||||
HnswElement entryPoint;
|
||||
|
||||
/* Get m and entry point */
|
||||
HnswGetMetaPageInfo(index, &m, &entryPoint);
|
||||
HnswElement entryPoint = HnswGetEntryPoint(index);
|
||||
|
||||
if (entryPoint == NULL)
|
||||
return NIL;
|
||||
@@ -32,11 +28,11 @@ GetScanItems(IndexScanDesc scan, Datum q)
|
||||
|
||||
for (int lc = entryPoint->level; lc >= 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
|
||||
w = HnswSearchLayer(q, ep, 1, lc, index, procinfo, collation, false, NULL);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
return HnswSearchLayer(q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
|
||||
return HnswSearchLayer(q, ep, hnsw_ef_search, 0, index, procinfo, collation, false, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -62,33 +58,6 @@ GetDimensions(Relation index)
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get scan value
|
||||
*/
|
||||
static Datum
|
||||
GetScanValue(IndexScanDesc scan)
|
||||
{
|
||||
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
|
||||
Datum value;
|
||||
|
||||
if (scan->orderByData->sk_flags & SK_ISNULL)
|
||||
value = PointerGetDatum(InitVector(GetDimensions(scan->indexRelation)));
|
||||
else
|
||||
{
|
||||
value = scan->orderByData->sk_argument;
|
||||
|
||||
/* Value should not be compressed or toasted */
|
||||
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
|
||||
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
|
||||
|
||||
/* Fine if normalization fails */
|
||||
if (so->normprocinfo != NULL)
|
||||
HnswNormValue(so->normprocinfo, so->collation, &value, NULL);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/*
|
||||
* Prepare for an index scan
|
||||
*/
|
||||
@@ -101,6 +70,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
|
||||
scan = RelationGetIndexScan(index, nkeys, norderbys);
|
||||
|
||||
so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
|
||||
so->buf = InvalidBuffer;
|
||||
so->first = true;
|
||||
so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
|
||||
"Hnsw scan temporary context",
|
||||
@@ -160,13 +130,20 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
if (scan->orderByData == NULL)
|
||||
elog(ERROR, "cannot scan hnsw index without order");
|
||||
|
||||
/* Requires MVCC-compliant snapshot as not able to maintain a pin */
|
||||
/* https://www.postgresql.org/docs/current/index-locking.html */
|
||||
if (!IsMVCCSnapshot(scan->xs_snapshot))
|
||||
elog(ERROR, "non-MVCC snapshots are not supported with hnsw");
|
||||
if (scan->orderByData->sk_flags & SK_ISNULL)
|
||||
value = PointerGetDatum(InitVector(GetDimensions(scan->indexRelation)));
|
||||
else
|
||||
{
|
||||
value = scan->orderByData->sk_argument;
|
||||
|
||||
/* Get scan value */
|
||||
value = GetScanValue(scan);
|
||||
/* Value should not be compressed or toasted */
|
||||
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
|
||||
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
|
||||
|
||||
/* Fine if normalization fails */
|
||||
if (so->normprocinfo != NULL)
|
||||
HnswNormValue(so->normprocinfo, so->collation, &value, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a shared lock. This allows vacuum to ensure no in-flight scans
|
||||
@@ -185,27 +162,40 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
while (list_length(so->w) > 0)
|
||||
{
|
||||
HnswCandidate *hc = llast(so->w);
|
||||
ItemPointer heaptid;
|
||||
ItemPointer tid;
|
||||
BlockNumber indexblkno;
|
||||
|
||||
/* Move to next element if no valid heap TIDs */
|
||||
/* Move to next element if no valid heap tids */
|
||||
if (list_length(hc->element->heaptids) == 0)
|
||||
{
|
||||
so->w = list_delete_last(so->w);
|
||||
continue;
|
||||
}
|
||||
|
||||
heaptid = llast(hc->element->heaptids);
|
||||
tid = llast(hc->element->heaptids);
|
||||
indexblkno = hc->element->blkno;
|
||||
|
||||
hc->element->heaptids = list_delete_last(hc->element->heaptids);
|
||||
|
||||
MemoryContextSwitchTo(oldCtx);
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
scan->xs_heaptid = *heaptid;
|
||||
scan->xs_heaptid = *tid;
|
||||
#else
|
||||
scan->xs_ctup.t_self = *heaptid;
|
||||
scan->xs_ctup.t_self = *tid;
|
||||
#endif
|
||||
|
||||
if (BufferIsValid(so->buf))
|
||||
ReleaseBuffer(so->buf);
|
||||
|
||||
/*
|
||||
* An index scan must maintain a pin on the index page holding the
|
||||
* item last returned by amgettuple
|
||||
*
|
||||
* https://www.postgresql.org/docs/current/index-locking.html
|
||||
*/
|
||||
so->buf = ReadBuffer(scan->indexRelation, indexblkno);
|
||||
|
||||
scan->xs_recheckorderby = false;
|
||||
return true;
|
||||
}
|
||||
@@ -222,6 +212,10 @@ hnswendscan(IndexScanDesc scan)
|
||||
{
|
||||
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
|
||||
|
||||
/* Release pin */
|
||||
if (BufferIsValid(so->buf))
|
||||
ReleaseBuffer(so->buf);
|
||||
|
||||
MemoryContextDelete(so->tmpCtx);
|
||||
|
||||
pfree(so);
|
||||
|
||||
@@ -38,12 +38,12 @@ HnswGetEfConstruction(Relation index)
|
||||
* Get proc
|
||||
*/
|
||||
FmgrInfo *
|
||||
HnswOptionalProcInfo(Relation index, uint16 procnum)
|
||||
HnswOptionalProcInfo(Relation rel, uint16 procnum)
|
||||
{
|
||||
if (!OidIsValid(index_getprocid(index, 1, procnum)))
|
||||
if (!OidIsValid(index_getprocid(rel, 1, procnum)))
|
||||
return NULL;
|
||||
|
||||
return index_getprocinfo(index, 1, procnum);
|
||||
return index_getprocinfo(rel, 1, procnum);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -117,6 +117,7 @@ HnswInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState *
|
||||
void
|
||||
HnswCommitBuffer(Buffer buf, GenericXLogState *state)
|
||||
{
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
@@ -142,17 +143,6 @@ HnswInitNeighbors(HnswElement element, int m)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Free neighbors
|
||||
*/
|
||||
static void
|
||||
HnswFreeNeighbors(HnswElement element)
|
||||
{
|
||||
for (int lc = 0; lc <= element->level; lc++)
|
||||
pfree(element->neighbors[lc].items);
|
||||
pfree(element->neighbors);
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate an element
|
||||
*/
|
||||
@@ -184,8 +174,10 @@ HnswInitElement(ItemPointer heaptid, int m, double ml, int maxLevel)
|
||||
void
|
||||
HnswFreeElement(HnswElement element)
|
||||
{
|
||||
HnswFreeNeighbors(element);
|
||||
list_free_deep(element->heaptids);
|
||||
for (int lc = 0; lc <= element->level; lc++)
|
||||
pfree(element->neighbors[lc].items);
|
||||
pfree(element->neighbors);
|
||||
pfree(element->vec);
|
||||
pfree(element);
|
||||
}
|
||||
@@ -218,43 +210,25 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the metapage info
|
||||
* Get the entry point
|
||||
*/
|
||||
void
|
||||
HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint)
|
||||
HnswElement
|
||||
HnswGetEntryPoint(Relation index)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
HnswMetaPage metap;
|
||||
HnswElement entryPoint = NULL;
|
||||
|
||||
buf = ReadBuffer(index, HNSW_METAPAGE_BLKNO);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
metap = HnswPageGetMeta(page);
|
||||
|
||||
if (m != NULL)
|
||||
*m = metap->m;
|
||||
|
||||
if (entryPoint != NULL)
|
||||
{
|
||||
if (BlockNumberIsValid(metap->entryBlkno))
|
||||
*entryPoint = HnswInitElementFromBlock(metap->entryBlkno, metap->entryOffno);
|
||||
else
|
||||
*entryPoint = NULL;
|
||||
}
|
||||
if (BlockNumberIsValid(metap->entryBlkno))
|
||||
entryPoint = HnswInitElementFromBlock(metap->entryBlkno, metap->entryOffno);
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the entry point
|
||||
*/
|
||||
HnswElement
|
||||
HnswGetEntryPoint(Relation index)
|
||||
{
|
||||
HnswElement entryPoint;
|
||||
|
||||
HnswGetMetaPageInfo(index, NULL, &entryPoint);
|
||||
|
||||
return entryPoint;
|
||||
}
|
||||
@@ -363,9 +337,10 @@ HnswSetNeighborTuple(HnswNeighborTuple ntup, HnswElement e, int m)
|
||||
* Load neighbors from page
|
||||
*/
|
||||
static void
|
||||
LoadNeighborsFromPage(HnswElement element, Relation index, Page page, int m)
|
||||
LoadNeighborsFromPage(HnswElement element, Relation index, Page page)
|
||||
{
|
||||
HnswNeighborTuple ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, element->neighborOffno));
|
||||
int m = HnswGetM(index);
|
||||
int neighborCount = (element->level + 2) * m;
|
||||
|
||||
Assert(HnswIsNeighborTuple(ntup));
|
||||
@@ -406,7 +381,7 @@ LoadNeighborsFromPage(HnswElement element, Relation index, Page page, int m)
|
||||
* Load neighbors
|
||||
*/
|
||||
void
|
||||
HnswLoadNeighbors(HnswElement element, Relation index, int m)
|
||||
HnswLoadNeighbors(HnswElement element, Relation index)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
@@ -415,7 +390,7 @@ HnswLoadNeighbors(HnswElement element, Relation index, int m)
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
|
||||
LoadNeighborsFromPage(element, index, page, m);
|
||||
LoadNeighborsFromPage(element, index, page);
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
@@ -568,7 +543,7 @@ AddToVisited(HTAB *v, HnswCandidate * hc, Relation index, bool *found)
|
||||
* Algorithm 2 from paper
|
||||
*/
|
||||
List *
|
||||
HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
|
||||
HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, bool inserting, HnswElement skipElement)
|
||||
{
|
||||
ListCell *lc2;
|
||||
|
||||
@@ -623,7 +598,7 @@ HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *pro
|
||||
break;
|
||||
|
||||
if (c->element->neighbors == NULL)
|
||||
HnswLoadNeighbors(c->element, index, m);
|
||||
HnswLoadNeighbors(c->element, index);
|
||||
|
||||
/* Get the neighborhood at layer lc */
|
||||
neighborhood = &c->element->neighbors[lc];
|
||||
@@ -648,6 +623,10 @@ HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *pro
|
||||
|
||||
Assert(!e->element->deleted);
|
||||
|
||||
/* Skip self for vacuuming update */
|
||||
if (skipElement != NULL && e->element->blkno == skipElement->blkno && e->element->offno == skipElement->offno)
|
||||
continue;
|
||||
|
||||
/* Make robust to issues */
|
||||
if (e->element->level < lc)
|
||||
continue;
|
||||
@@ -934,10 +913,10 @@ HnswUpdateConnection(HnswElement element, HnswCandidate * hc, int m, int lc, int
|
||||
}
|
||||
|
||||
/*
|
||||
* Remove elements being deleted or skipped
|
||||
* Remove elements being deleted
|
||||
*/
|
||||
static List *
|
||||
RemoveElements(List *w, HnswElement skipElement)
|
||||
RemoveElementsBeingDeleted(List *w)
|
||||
{
|
||||
ListCell *lc2;
|
||||
List *w2 = NIL;
|
||||
@@ -946,10 +925,6 @@ RemoveElements(List *w, HnswElement skipElement)
|
||||
{
|
||||
HnswCandidate *hc = (HnswCandidate *) lfirst(lc2);
|
||||
|
||||
/* Skip self for vacuuming update */
|
||||
if (skipElement != NULL && hc->element->blkno == skipElement->blkno && hc->element->offno == skipElement->offno)
|
||||
continue;
|
||||
|
||||
if (list_length(hc->element->heaptids) != 0)
|
||||
w2 = lappend(w2, hc);
|
||||
}
|
||||
@@ -981,34 +956,27 @@ HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, F
|
||||
/* 1st phase: greedy search to insert level */
|
||||
for (int lc = entryLevel; lc >= level + 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
|
||||
w = HnswSearchLayer(q, ep, 1, lc, index, procinfo, collation, true, skipElement);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
if (level > entryLevel)
|
||||
level = entryLevel;
|
||||
|
||||
/* Add one for existing element */
|
||||
if (existing)
|
||||
efConstruction++;
|
||||
|
||||
/* 2nd phase */
|
||||
for (int lc = level; lc >= 0; lc--)
|
||||
{
|
||||
int lm = HnswGetLayerM(m, lc);
|
||||
List *neighbors;
|
||||
List *lw;
|
||||
|
||||
w = HnswSearchLayer(q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
|
||||
w = HnswSearchLayer(q, ep, efConstruction, lc, index, procinfo, collation, true, skipElement);
|
||||
|
||||
/* Elements being deleted or skipped can help with search */
|
||||
/* Elements being deleted can help with search */
|
||||
/* but should be removed before selecting neighbors */
|
||||
if (index != NULL)
|
||||
lw = RemoveElements(w, skipElement);
|
||||
else
|
||||
lw = w;
|
||||
w = RemoveElementsBeingDeleted(w);
|
||||
|
||||
neighbors = SelectNeighbors(lw, lm, lc, procinfo, collation, NULL);
|
||||
neighbors = SelectNeighbors(w, lm, lc, procinfo, collation, NULL);
|
||||
|
||||
AddConnections(element, neighbors, lm, lc);
|
||||
|
||||
|
||||
@@ -128,7 +128,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
|
||||
blkno = HnswPageGetOpaque(page)->nextblkno;
|
||||
|
||||
if (updated)
|
||||
{
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
}
|
||||
else
|
||||
GenericXLogAbort(state);
|
||||
|
||||
@@ -226,6 +229,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
@@ -326,10 +330,7 @@ RepairGraph(HnswVacuumState * vacuumstate)
|
||||
BufferAccessStrategy bas = vacuumstate->bas;
|
||||
BlockNumber blkno = HNSW_HEAD_BLKNO;
|
||||
|
||||
/*
|
||||
* Wait for inserts to complete. Inserts before this point may have
|
||||
* neighbors about to be deleted. Inserts after this point will not.
|
||||
*/
|
||||
/* Wait for inserts to complete */
|
||||
LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
|
||||
UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
|
||||
|
||||
@@ -442,11 +443,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
Relation index = vacuumstate->index;
|
||||
BufferAccessStrategy bas = vacuumstate->bas;
|
||||
|
||||
/*
|
||||
* Wait for index scans to complete. Scans before this point may contain
|
||||
* tuples about to be deleted. Scans after this point will not, since the
|
||||
* graph has been repaired.
|
||||
*/
|
||||
/* Wait for selects to complete */
|
||||
LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
|
||||
UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
|
||||
|
||||
@@ -543,6 +540,9 @@ MarkDeleted(HnswVacuumState * vacuumstate)
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(buf);
|
||||
if (nbuf != buf)
|
||||
MarkBufferDirty(nbuf);
|
||||
GenericXLogFinish(state);
|
||||
if (nbuf != buf)
|
||||
UnlockReleaseBuffer(nbuf);
|
||||
@@ -582,6 +582,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
|
||||
vacuumstate->stats = stats;
|
||||
vacuumstate->callback = callback;
|
||||
vacuumstate->callback_state = callback_state;
|
||||
vacuumstate->m = HnswGetM(index);
|
||||
vacuumstate->efConstruction = HnswGetEfConstruction(index);
|
||||
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
|
||||
vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
|
||||
@@ -591,9 +592,6 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
|
||||
"Hnsw vacuum temporary context",
|
||||
ALLOCSET_DEFAULT_SIZES);
|
||||
|
||||
/* Get m from metapage */
|
||||
HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
|
||||
|
||||
/* Create hash table */
|
||||
hash_ctl.keysize = sizeof(ItemPointerData);
|
||||
hash_ctl.entrysize = sizeof(ItemPointerData);
|
||||
@@ -613,6 +611,67 @@ FreeVacuumState(HnswVacuumState * vacuumstate)
|
||||
MemoryContextDelete(vacuumstate->tmpCtx);
|
||||
}
|
||||
|
||||
/*
|
||||
* Print graph
|
||||
*/
|
||||
#ifdef HNSW_DEBUG
|
||||
static void
|
||||
PrintGraph(HnswVacuumState * vacuumstate)
|
||||
{
|
||||
BlockNumber blkno = HNSW_HEAD_BLKNO;
|
||||
Relation index = vacuumstate->index;
|
||||
|
||||
while (BlockNumberIsValid(blkno))
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
buf = ReadBuffer(index, blkno);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
maxoffno = PageGetMaxOffsetNumber(page);
|
||||
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
HnswElement element;
|
||||
|
||||
/* Skip neighbor tuples */
|
||||
if (!HnswIsElementTuple(etup))
|
||||
continue;
|
||||
|
||||
/* Skip deleted tuples */
|
||||
if (etup->deleted)
|
||||
continue;
|
||||
|
||||
element = HnswInitElementFromBlock(blkno, offno);
|
||||
HnswLoadElementFromTuple(element, etup, false, true);
|
||||
HnswLoadNeighbors(element, index);
|
||||
|
||||
elog(INFO, "element (%d,%d)", element->blkno, element->offno);
|
||||
|
||||
for (int lc = element->level; lc >= 0; lc--)
|
||||
{
|
||||
HnswNeighborArray *neighbors = &element->neighbors[lc];
|
||||
|
||||
for (int i = 0; i < neighbors->length; i++)
|
||||
{
|
||||
HnswElement e = neighbors->items[i].element;
|
||||
|
||||
elog(INFO, "%d: (%d,%d)", lc, e->blkno, e->offno);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blkno = HnswPageGetOpaque(page)->nextblkno;
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Bulk delete tuples from the index
|
||||
*/
|
||||
@@ -633,6 +692,10 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
|
||||
/* Pass 3: Mark as deleted */
|
||||
MarkDeleted(&vacuumstate);
|
||||
|
||||
#ifdef HNSW_DEBUG
|
||||
PrintGraph(&vacuumstate);
|
||||
#endif
|
||||
|
||||
FreeVacuumState(&vacuumstate);
|
||||
|
||||
return vacuumstate.stats;
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
#include "ivfflat.h"
|
||||
#include "miscadmin.h"
|
||||
#include "storage/bufmgr.h"
|
||||
#include "tcop/tcopprot.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "tcop/tcopprot.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 140000
|
||||
#include "utils/backend_progress.h"
|
||||
@@ -506,30 +506,29 @@ CreateListPages(Relation index, VectorArray centers, int dimensions,
|
||||
Buffer buf;
|
||||
Page page;
|
||||
GenericXLogState *state;
|
||||
Size listSize;
|
||||
OffsetNumber offno;
|
||||
Size itemsz;
|
||||
IvfflatList list;
|
||||
|
||||
listSize = MAXALIGN(IVFFLAT_LIST_SIZE(dimensions));
|
||||
list = palloc(listSize);
|
||||
itemsz = MAXALIGN(IVFFLAT_LIST_SIZE(dimensions));
|
||||
list = palloc(itemsz);
|
||||
|
||||
buf = IvfflatNewBuffer(index, forkNum);
|
||||
IvfflatInitRegisterPage(index, &buf, &page, &state);
|
||||
|
||||
for (int i = 0; i < lists; i++)
|
||||
{
|
||||
OffsetNumber offno;
|
||||
|
||||
/* Load list */
|
||||
list->startPage = InvalidBlockNumber;
|
||||
list->insertPage = InvalidBlockNumber;
|
||||
memcpy(&list->center, VectorArrayGet(centers, i), VECTOR_SIZE(dimensions));
|
||||
|
||||
/* Ensure free space */
|
||||
if (PageGetFreeSpace(page) < listSize)
|
||||
if (PageGetFreeSpace(page) < itemsz)
|
||||
IvfflatAppendPage(index, &buf, &page, &state, forkNum);
|
||||
|
||||
/* Add the item */
|
||||
offno = PageAddItem(page, (Item) list, listSize, InvalidOffsetNumber, false, false);
|
||||
offno = PageAddItem(page, (Item) list, itemsz, InvalidOffsetNumber, false, false);
|
||||
if (offno == InvalidOffsetNumber)
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
int lists;
|
||||
double ratio;
|
||||
double spc_seq_page_cost;
|
||||
Relation index;
|
||||
Relation indexRel;
|
||||
#if PG_VERSION_NUM < 120000
|
||||
List *qinfos;
|
||||
#endif
|
||||
@@ -89,9 +89,9 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
|
||||
MemSet(&costs, 0, sizeof(costs));
|
||||
|
||||
index = index_open(path->indexinfo->indexoid, NoLock);
|
||||
IvfflatGetMetaPageInfo(index, &lists, NULL);
|
||||
index_close(index, NoLock);
|
||||
indexRel = index_open(path->indexinfo->indexoid, NoLock);
|
||||
lists = IvfflatGetLists(indexRel);
|
||||
index_close(indexRel, NoLock);
|
||||
|
||||
/* Get the ratio of lists that we need to visit */
|
||||
ratio = ((double) ivfflat_probes) / lists;
|
||||
|
||||
@@ -244,10 +244,8 @@ typedef struct IvfflatScanList
|
||||
typedef struct IvfflatScanOpaqueData
|
||||
{
|
||||
int probes;
|
||||
int dimensions;
|
||||
bool first;
|
||||
Buffer buf;
|
||||
ItemPointerData heaptid;
|
||||
|
||||
/* Sorting */
|
||||
Tuplesortstate *sortstate;
|
||||
@@ -277,10 +275,9 @@ VectorArray VectorArrayInit(int maxlen, int dimensions);
|
||||
void VectorArrayFree(VectorArray arr);
|
||||
void PrintVectorArray(char *msg, VectorArray arr);
|
||||
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
|
||||
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
|
||||
FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);
|
||||
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
|
||||
int IvfflatGetLists(Relation index);
|
||||
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
|
||||
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
|
||||
void IvfflatCommitBuffer(Buffer buf, GenericXLogState *state);
|
||||
void IvfflatAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state, ForkNumber forkNum);
|
||||
|
||||
@@ -11,37 +11,36 @@
|
||||
* Find the list that minimizes the distance function
|
||||
*/
|
||||
static void
|
||||
FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo * listInfo)
|
||||
FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo * listInfo)
|
||||
{
|
||||
Buffer cbuf;
|
||||
Page cpage;
|
||||
IvfflatList list;
|
||||
double distance;
|
||||
double minDistance = DBL_MAX;
|
||||
BlockNumber nextblkno = IVFFLAT_HEAD_BLKNO;
|
||||
FmgrInfo *procinfo;
|
||||
Oid collation;
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
/* Avoid compiler warning */
|
||||
listInfo->blkno = nextblkno;
|
||||
listInfo->offno = FirstOffsetNumber;
|
||||
|
||||
procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
|
||||
collation = index->rd_indcollation[0];
|
||||
procinfo = index_getprocinfo(rel, 1, IVFFLAT_DISTANCE_PROC);
|
||||
collation = rel->rd_indcollation[0];
|
||||
|
||||
/* Search all list pages */
|
||||
while (BlockNumberIsValid(nextblkno))
|
||||
{
|
||||
Buffer cbuf;
|
||||
Page cpage;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
cbuf = ReadBuffer(index, nextblkno);
|
||||
cbuf = ReadBuffer(rel, nextblkno);
|
||||
LockBuffer(cbuf, BUFFER_LOCK_SHARE);
|
||||
cpage = BufferGetPage(cbuf);
|
||||
maxoffno = PageGetMaxOffsetNumber(cpage);
|
||||
|
||||
for (OffsetNumber offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
IvfflatList list;
|
||||
double distance;
|
||||
|
||||
list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno));
|
||||
distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, values[0], PointerGetDatum(&list->center)));
|
||||
|
||||
@@ -64,7 +63,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
|
||||
* Insert a tuple into the index
|
||||
*/
|
||||
static void
|
||||
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
|
||||
InsertTuple(Relation rel, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
|
||||
{
|
||||
IndexTuple itup;
|
||||
Datum value;
|
||||
@@ -81,33 +80,33 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
|
||||
value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
|
||||
|
||||
/* Normalize if needed */
|
||||
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
||||
normprocinfo = IvfflatOptionalProcInfo(rel, IVFFLAT_NORM_PROC);
|
||||
if (normprocinfo != NULL)
|
||||
{
|
||||
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value, NULL))
|
||||
if (!IvfflatNormValue(normprocinfo, rel->rd_indcollation[0], &value, NULL))
|
||||
return;
|
||||
}
|
||||
|
||||
/* Find the insert page - sets the page and list info */
|
||||
FindInsertPage(index, values, &insertPage, &listInfo);
|
||||
FindInsertPage(rel, values, &insertPage, &listInfo);
|
||||
Assert(BlockNumberIsValid(insertPage));
|
||||
originalInsertPage = insertPage;
|
||||
|
||||
/* Form tuple */
|
||||
itup = index_form_tuple(RelationGetDescr(index), &value, isnull);
|
||||
itup = index_form_tuple(RelationGetDescr(rel), &value, isnull);
|
||||
itup->t_tid = *heap_tid;
|
||||
|
||||
/* Get tuple size */
|
||||
itemsz = MAXALIGN(IndexTupleSize(itup));
|
||||
Assert(itemsz <= BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(IvfflatPageOpaqueData)) - sizeof(ItemIdData));
|
||||
Assert(itemsz <= BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(IvfflatPageOpaqueData)));
|
||||
|
||||
/* Find a page to insert the item */
|
||||
for (;;)
|
||||
{
|
||||
buf = ReadBuffer(index, insertPage);
|
||||
buf = ReadBuffer(rel, insertPage);
|
||||
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
|
||||
|
||||
state = GenericXLogStart(index);
|
||||
state = GenericXLogStart(rel);
|
||||
page = GenericXLogRegisterBuffer(state, buf, 0);
|
||||
|
||||
if (PageGetFreeSpace(page) >= itemsz)
|
||||
@@ -127,9 +126,9 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
|
||||
Page newpage;
|
||||
|
||||
/* Add a new page */
|
||||
LockRelationForExtension(index, ExclusiveLock);
|
||||
newbuf = IvfflatNewBuffer(index, MAIN_FORKNUM);
|
||||
UnlockRelationForExtension(index, ExclusiveLock);
|
||||
LockRelationForExtension(rel, ExclusiveLock);
|
||||
newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
|
||||
UnlockRelationForExtension(rel, ExclusiveLock);
|
||||
|
||||
/* Init new page */
|
||||
newpage = GenericXLogRegisterBuffer(state, newbuf, GENERIC_XLOG_FULL_IMAGE);
|
||||
@@ -142,13 +141,15 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
|
||||
IvfflatPageGetOpaque(page)->nextblkno = insertPage;
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(newbuf);
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
|
||||
/* Unlock previous buffer */
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
/* Prepare new buffer */
|
||||
state = GenericXLogStart(index);
|
||||
state = GenericXLogStart(rel);
|
||||
buf = newbuf;
|
||||
page = GenericXLogRegisterBuffer(state, buf, 0);
|
||||
break;
|
||||
@@ -157,13 +158,13 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
|
||||
|
||||
/* Add to next offset */
|
||||
if (PageAddItem(page, (Item) itup, itemsz, InvalidOffsetNumber, false, false) == InvalidOffsetNumber)
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(index));
|
||||
elog(ERROR, "failed to add index item to \"%s\"", RelationGetRelationName(rel));
|
||||
|
||||
IvfflatCommitBuffer(buf, state);
|
||||
|
||||
/* Update the insert page */
|
||||
if (insertPage != originalInsertPage)
|
||||
IvfflatUpdateList(index, listInfo, insertPage, originalInsertPage, InvalidBlockNumber, MAIN_FORKNUM);
|
||||
IvfflatUpdateList(rel, listInfo, insertPage, originalInsertPage, InvalidBlockNumber, MAIN_FORKNUM);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -17,6 +17,10 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
|
||||
FmgrInfo *procinfo;
|
||||
Oid collation;
|
||||
int64 j;
|
||||
double distance;
|
||||
double sum;
|
||||
double choice;
|
||||
Vector *vec;
|
||||
float *weight = palloc(samples->length * sizeof(float));
|
||||
int numCenters = centers->maxlen;
|
||||
int numSamples = samples->length;
|
||||
@@ -29,21 +33,17 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
|
||||
centers->length++;
|
||||
|
||||
for (j = 0; j < numSamples; j++)
|
||||
weight[j] = FLT_MAX;
|
||||
weight[j] = DBL_MAX;
|
||||
|
||||
for (int i = 0; i < numCenters; i++)
|
||||
{
|
||||
double sum;
|
||||
double choice;
|
||||
|
||||
CHECK_FOR_INTERRUPTS();
|
||||
|
||||
sum = 0.0;
|
||||
|
||||
for (j = 0; j < numSamples; j++)
|
||||
{
|
||||
Vector *vec = VectorArrayGet(samples, j);
|
||||
double distance;
|
||||
vec = VectorArrayGet(samples, j);
|
||||
|
||||
/* Only need to compute distance for new center */
|
||||
/* TODO Use triangle inequality to reduce distance calculations */
|
||||
@@ -112,6 +112,7 @@ CompareVectors(const void *a, const void *b)
|
||||
static void
|
||||
QuickCenters(Relation index, VectorArray samples, VectorArray centers)
|
||||
{
|
||||
Vector *vec;
|
||||
int dimensions = centers->dim;
|
||||
Oid collation = index->rd_indcollation[0];
|
||||
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
|
||||
@@ -122,7 +123,7 @@ QuickCenters(Relation index, VectorArray samples, VectorArray centers)
|
||||
qsort(samples->items, samples->length, VECTOR_SIZE(samples->dim), CompareVectors);
|
||||
for (int i = 0; i < samples->length; i++)
|
||||
{
|
||||
Vector *vec = VectorArrayGet(samples, i);
|
||||
vec = VectorArrayGet(samples, i);
|
||||
|
||||
if (i == 0 || CompareVectors(vec, VectorArrayGet(samples, i - 1)) != 0)
|
||||
{
|
||||
@@ -135,7 +136,7 @@ QuickCenters(Relation index, VectorArray samples, VectorArray centers)
|
||||
/* Fill remaining with random data */
|
||||
while (centers->length < centers->maxlen)
|
||||
{
|
||||
Vector *vec = VectorArrayGet(centers, centers->length);
|
||||
vec = VectorArrayGet(centers, centers->length);
|
||||
|
||||
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
|
||||
vec->dim = dimensions;
|
||||
@@ -167,6 +168,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
Oid collation;
|
||||
Vector *vec;
|
||||
Vector *newCenter;
|
||||
int iteration;
|
||||
int64 j;
|
||||
int64 k;
|
||||
int dimensions = centers->dim;
|
||||
@@ -180,6 +182,14 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
float *s;
|
||||
float *halfcdist;
|
||||
float *newcdist;
|
||||
int changes;
|
||||
double minDistance;
|
||||
int closestCenter;
|
||||
double distance;
|
||||
bool rj;
|
||||
bool rjreset;
|
||||
double dxcx;
|
||||
double dxc;
|
||||
|
||||
/* Calculate allocation sizes */
|
||||
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->dim);
|
||||
@@ -237,14 +247,14 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
/* Assign each x to its closest initial center c(x) = argmin d(x,c) */
|
||||
for (j = 0; j < numSamples; j++)
|
||||
{
|
||||
float minDistance = FLT_MAX;
|
||||
int closestCenter = 0;
|
||||
minDistance = DBL_MAX;
|
||||
closestCenter = 0;
|
||||
|
||||
/* Find closest center */
|
||||
for (k = 0; k < numCenters; k++)
|
||||
{
|
||||
/* TODO Use Lemma 1 in k-means++ initialization */
|
||||
float distance = lowerBound[j * numCenters + k];
|
||||
distance = lowerBound[j * numCenters + k];
|
||||
|
||||
if (distance < minDistance)
|
||||
{
|
||||
@@ -258,14 +268,13 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
}
|
||||
|
||||
/* Give 500 iterations to converge */
|
||||
for (int iteration = 0; iteration < 500; iteration++)
|
||||
for (iteration = 0; iteration < 500; iteration++)
|
||||
{
|
||||
int changes = 0;
|
||||
bool rjreset;
|
||||
|
||||
/* Can take a while, so ensure we can interrupt */
|
||||
CHECK_FOR_INTERRUPTS();
|
||||
|
||||
changes = 0;
|
||||
|
||||
/* Step 1: For all centers, compute distance */
|
||||
for (j = 0; j < numCenters; j++)
|
||||
{
|
||||
@@ -273,8 +282,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
for (k = j + 1; k < numCenters; k++)
|
||||
{
|
||||
float distance = 0.5 * DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, k))));
|
||||
|
||||
distance = 0.5 * DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, k))));
|
||||
halfcdist[j * numCenters + k] = distance;
|
||||
halfcdist[k * numCenters + j] = distance;
|
||||
}
|
||||
@@ -283,12 +291,10 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
/* For all centers c, compute s(c) */
|
||||
for (j = 0; j < numCenters; j++)
|
||||
{
|
||||
float minDistance = FLT_MAX;
|
||||
minDistance = DBL_MAX;
|
||||
|
||||
for (k = 0; k < numCenters; k++)
|
||||
{
|
||||
float distance;
|
||||
|
||||
if (j == k)
|
||||
continue;
|
||||
|
||||
@@ -304,8 +310,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
for (j = 0; j < numSamples; j++)
|
||||
{
|
||||
bool rj;
|
||||
|
||||
/* Step 2: Identify all points x such that u(x) <= s(c(x)) */
|
||||
if (upperBound[j] <= s[closestCenters[j]])
|
||||
continue;
|
||||
@@ -314,8 +318,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
for (k = 0; k < numCenters; k++)
|
||||
{
|
||||
float dxcx;
|
||||
|
||||
/* Step 3: For all remaining points x and centers c */
|
||||
if (k == closestCenters[j])
|
||||
continue;
|
||||
@@ -345,7 +347,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
/* Step 3b */
|
||||
if (dxcx > lowerBound[j * numCenters + k] || dxcx > halfcdist[closestCenters[j] * numCenters + k])
|
||||
{
|
||||
float dxc = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, k))));
|
||||
dxc = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, k))));
|
||||
|
||||
/* d(x,c) calculated */
|
||||
lowerBound[j * numCenters + k] = dxc;
|
||||
@@ -359,6 +361,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
changes++;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -375,8 +378,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
for (j = 0; j < numSamples; j++)
|
||||
{
|
||||
int closestCenter;
|
||||
|
||||
vec = VectorArrayGet(samples, j);
|
||||
closestCenter = closestCenters[j];
|
||||
|
||||
@@ -425,7 +426,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
{
|
||||
for (k = 0; k < numCenters; k++)
|
||||
{
|
||||
float distance = lowerBound[j * numCenters + k] - newcdist[k];
|
||||
distance = lowerBound[j * numCenters + k] - newcdist[k];
|
||||
|
||||
if (distance < 0)
|
||||
distance = 0;
|
||||
@@ -441,7 +442,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
/* Step 7 */
|
||||
for (j = 0; j < numCenters; j++)
|
||||
VectorArraySet(centers, j, VectorArrayGet(newCenters, j));
|
||||
memcpy(VectorArrayGet(centers, j), VectorArrayGet(newCenters, j), VECTOR_SIZE(dimensions));
|
||||
|
||||
if (changes == 0 && iteration != 0)
|
||||
break;
|
||||
@@ -464,6 +465,9 @@ static void
|
||||
CheckCenters(Relation index, VectorArray centers)
|
||||
{
|
||||
FmgrInfo *normprocinfo;
|
||||
Oid collation;
|
||||
Vector *vec;
|
||||
double norm;
|
||||
|
||||
if (centers->length != centers->maxlen)
|
||||
elog(ERROR, "Not enough centers. Please report a bug.");
|
||||
@@ -471,7 +475,7 @@ CheckCenters(Relation index, VectorArray centers)
|
||||
/* Ensure no NaN or infinite values */
|
||||
for (int i = 0; i < centers->length; i++)
|
||||
{
|
||||
Vector *vec = VectorArrayGet(centers, i);
|
||||
vec = VectorArrayGet(centers, i);
|
||||
|
||||
for (int j = 0; j < vec->dim; j++)
|
||||
{
|
||||
@@ -497,12 +501,11 @@ CheckCenters(Relation index, VectorArray centers)
|
||||
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
|
||||
if (normprocinfo != NULL)
|
||||
{
|
||||
Oid collation = index->rd_indcollation[0];
|
||||
collation = index->rd_indcollation[0];
|
||||
|
||||
for (int i = 0; i < centers->length; i++)
|
||||
{
|
||||
double norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(VectorArrayGet(centers, i))));
|
||||
|
||||
norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(VectorArrayGet(centers, i))));
|
||||
if (norm == 0)
|
||||
elog(ERROR, "Zero norm detected. Please report a bug.");
|
||||
}
|
||||
|
||||
125
src/ivfscan.c
125
src/ivfscan.c
@@ -31,36 +31,36 @@ CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg)
|
||||
static void
|
||||
GetScanLists(IndexScanDesc scan, Datum value)
|
||||
{
|
||||
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
|
||||
Buffer cbuf;
|
||||
Page cpage;
|
||||
IvfflatList list;
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno;
|
||||
BlockNumber nextblkno = IVFFLAT_HEAD_BLKNO;
|
||||
int listCount = 0;
|
||||
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
|
||||
double distance;
|
||||
IvfflatScanList *scanlist;
|
||||
double maxDistance = DBL_MAX;
|
||||
|
||||
/* Search all list pages */
|
||||
while (BlockNumberIsValid(nextblkno))
|
||||
{
|
||||
Buffer cbuf;
|
||||
Page cpage;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
cbuf = ReadBuffer(scan->indexRelation, nextblkno);
|
||||
LockBuffer(cbuf, BUFFER_LOCK_SHARE);
|
||||
cpage = BufferGetPage(cbuf);
|
||||
|
||||
maxoffno = PageGetMaxOffsetNumber(cpage);
|
||||
|
||||
for (OffsetNumber offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
IvfflatList list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno));
|
||||
double distance;
|
||||
list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno));
|
||||
|
||||
/* Use procinfo from the index instead of scan key for performance */
|
||||
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
|
||||
|
||||
if (listCount < so->probes)
|
||||
{
|
||||
IvfflatScanList *scanlist;
|
||||
|
||||
scanlist = &so->lists[listCount];
|
||||
scanlist->startPage = list->startPage;
|
||||
scanlist->distance = distance;
|
||||
@@ -75,8 +75,6 @@ GetScanLists(IndexScanDesc scan, Datum value)
|
||||
}
|
||||
else if (distance < maxDistance)
|
||||
{
|
||||
IvfflatScanList *scanlist;
|
||||
|
||||
/* Remove */
|
||||
scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
|
||||
|
||||
@@ -103,6 +101,14 @@ static void
|
||||
GetScanItems(IndexScanDesc scan, Datum value)
|
||||
{
|
||||
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
|
||||
Buffer buf;
|
||||
Page page;
|
||||
IndexTuple itup;
|
||||
BlockNumber searchPage;
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno;
|
||||
Datum datum;
|
||||
bool isnull;
|
||||
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
|
||||
double tuples = 0;
|
||||
|
||||
@@ -122,32 +128,19 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
/* Search closest probes lists */
|
||||
while (!pairingheap_is_empty(so->listQueue))
|
||||
{
|
||||
BlockNumber searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
|
||||
searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
|
||||
|
||||
/* Search all entry pages for list */
|
||||
while (BlockNumberIsValid(searchPage))
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, bas);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
maxoffno = PageGetMaxOffsetNumber(page);
|
||||
|
||||
for (OffsetNumber offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
IndexTuple itup;
|
||||
Datum datum;
|
||||
bool isnull;
|
||||
ItemId itemid = PageGetItemId(page, offno);
|
||||
|
||||
/* Skip dead tuples */
|
||||
if (scan->ignore_killed_tuples && ItemIdIsDead(itemid))
|
||||
continue;
|
||||
|
||||
itup = (IndexTuple) PageGetItem(page, itemid);
|
||||
itup = (IndexTuple) PageGetItem(page, PageGetItemId(page, offno));
|
||||
datum = index_getattr(itup, 1, tupdesc, &isnull);
|
||||
|
||||
/*
|
||||
@@ -188,52 +181,26 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
}
|
||||
|
||||
/*
|
||||
* Mark prior tuple as dead
|
||||
* Get dimensions from metapage
|
||||
*/
|
||||
static void
|
||||
MarkPriorTupleDead(IndexScanDesc scan)
|
||||
static int
|
||||
GetDimensions(Relation index)
|
||||
{
|
||||
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
|
||||
Buffer buf = so->buf;
|
||||
Buffer buf;
|
||||
Page page;
|
||||
OffsetNumber maxoffno;
|
||||
IvfflatMetaPage metap;
|
||||
int dimensions;
|
||||
|
||||
/* Safety check */
|
||||
if (!BufferIsValid(so->buf) || !ItemPointerIsValid(&so->heaptid))
|
||||
return;
|
||||
|
||||
/* Only a shared locked is needed for ItemIdMarkDead */
|
||||
buf = ReadBuffer(index, IVFFLAT_METAPAGE_BLKNO);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
maxoffno = PageGetMaxOffsetNumber(page);
|
||||
metap = IvfflatPageGetMeta(page);
|
||||
|
||||
for (OffsetNumber offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
|
||||
{
|
||||
ItemId itemid = PageGetItemId(page, offno);
|
||||
IndexTuple itup = (IndexTuple) PageGetItem(page, itemid);
|
||||
dimensions = metap->dimensions;
|
||||
|
||||
/*
|
||||
* Find tuple. Since buffer has been pinned, tuple cannot have been
|
||||
* vacuumed (and heap TID reused).
|
||||
*/
|
||||
if (ItemPointerEquals(&itup->t_tid, &so->heaptid))
|
||||
{
|
||||
/*
|
||||
* Make sure tuple has not already been marked dead to avoid extra
|
||||
* WAL if wal_log_hints or data checksums enabled
|
||||
*/
|
||||
if (!ItemIdIsDead(itemid))
|
||||
{
|
||||
ItemIdMarkDead(itemid);
|
||||
MarkBufferDirtyHint(buf, true);
|
||||
}
|
||||
UnlockReleaseBuffer(buf);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Unlock buffer */
|
||||
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -245,7 +212,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
|
||||
IndexScanDesc scan;
|
||||
IvfflatScanOpaque so;
|
||||
int lists;
|
||||
int dimensions;
|
||||
AttrNumber attNums[] = {1};
|
||||
Oid sortOperators[] = {Float8LessOperator};
|
||||
Oid sortCollations[] = {InvalidOid};
|
||||
@@ -253,9 +219,7 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
|
||||
int probes = ivfflat_probes;
|
||||
|
||||
scan = RelationGetIndexScan(index, nkeys, norderbys);
|
||||
|
||||
/* Get lists and dimensions from metapage */
|
||||
IvfflatGetMetaPageInfo(index, &lists, &dimensions);
|
||||
lists = IvfflatGetLists(scan->indexRelation);
|
||||
|
||||
if (probes > lists)
|
||||
probes = lists;
|
||||
@@ -263,9 +227,7 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
|
||||
so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + probes * sizeof(IvfflatScanList));
|
||||
so->buf = InvalidBuffer;
|
||||
so->first = true;
|
||||
ItemPointerSetInvalid(&so->heaptid);
|
||||
so->probes = probes;
|
||||
so->dimensions = dimensions;
|
||||
|
||||
/* Set support functions */
|
||||
so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
|
||||
@@ -279,7 +241,7 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
|
||||
so->tupdesc = CreateTemplateTupleDesc(3, false);
|
||||
#endif
|
||||
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
|
||||
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
|
||||
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
|
||||
TupleDescInitEntry(so->tupdesc, (AttrNumber) 3, "indexblkno", INT4OID, -1, 0);
|
||||
|
||||
/* Prep sort */
|
||||
@@ -312,7 +274,6 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
|
||||
#endif
|
||||
|
||||
so->first = true;
|
||||
ItemPointerSetInvalid(&so->heaptid);
|
||||
pairingheap_reset(so->listQueue);
|
||||
|
||||
if (keys && scan->numberOfKeys > 0)
|
||||
@@ -348,7 +309,7 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
elog(ERROR, "cannot scan ivfflat index without order");
|
||||
|
||||
if (scan->orderByData->sk_flags & SK_ISNULL)
|
||||
value = PointerGetDatum(InitVector(so->dimensions));
|
||||
value = PointerGetDatum(InitVector(GetDimensions(scan->indexRelation)));
|
||||
else
|
||||
{
|
||||
value = scan->orderByData->sk_argument;
|
||||
@@ -370,28 +331,18 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
|
||||
if (value != scan->orderByData->sk_argument)
|
||||
pfree(DatumGetPointer(value));
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Mark prior tuple as dead */
|
||||
if (scan->kill_prior_tuple)
|
||||
MarkPriorTupleDead(scan);
|
||||
}
|
||||
|
||||
if (tuplesort_gettupleslot(so->sortstate, true, false, so->slot, NULL))
|
||||
{
|
||||
ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
|
||||
ItemPointer tid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
|
||||
BlockNumber indexblkno = DatumGetInt32(slot_getattr(so->slot, 3, &so->isnull));
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
scan->xs_heaptid = *heaptid;
|
||||
scan->xs_heaptid = *tid;
|
||||
#else
|
||||
scan->xs_ctup.t_self = *heaptid;
|
||||
scan->xs_ctup.t_self = *tid;
|
||||
#endif
|
||||
|
||||
/* Keep track of info needed to mark tuple as dead */
|
||||
so->heaptid = *heaptid;
|
||||
|
||||
/* Unpin buffer */
|
||||
if (BufferIsValid(so->buf))
|
||||
ReleaseBuffer(so->buf);
|
||||
|
||||
|
||||
@@ -57,12 +57,12 @@ IvfflatGetLists(Relation index)
|
||||
* Get proc
|
||||
*/
|
||||
FmgrInfo *
|
||||
IvfflatOptionalProcInfo(Relation index, uint16 procnum)
|
||||
IvfflatOptionalProcInfo(Relation rel, uint16 procnum)
|
||||
{
|
||||
if (!OidIsValid(index_getprocid(index, 1, procnum)))
|
||||
if (!OidIsValid(index_getprocid(rel, 1, procnum)))
|
||||
return NULL;
|
||||
|
||||
return index_getprocinfo(index, 1, procnum);
|
||||
return index_getprocinfo(rel, 1, procnum);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -136,6 +136,7 @@ IvfflatInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogStat
|
||||
void
|
||||
IvfflatCommitBuffer(Buffer buf, GenericXLogState *state)
|
||||
{
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
@@ -159,6 +160,8 @@ IvfflatAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **st
|
||||
IvfflatInitPage(newbuf, newpage);
|
||||
|
||||
/* Commit */
|
||||
MarkBufferDirty(*buf);
|
||||
MarkBufferDirty(newbuf);
|
||||
GenericXLogFinish(*state);
|
||||
|
||||
/* Unlock */
|
||||
@@ -169,29 +172,6 @@ IvfflatAppendPage(Relation index, Buffer *buf, Page *page, GenericXLogState **st
|
||||
*buf = newbuf;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the metapage info
|
||||
*/
|
||||
void
|
||||
IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
IvfflatMetaPage metap;
|
||||
|
||||
buf = ReadBuffer(index, IVFFLAT_METAPAGE_BLKNO);
|
||||
LockBuffer(buf, BUFFER_LOCK_SHARE);
|
||||
page = BufferGetPage(buf);
|
||||
metap = IvfflatPageGetMeta(page);
|
||||
|
||||
*lists = metap->lists;
|
||||
|
||||
if (dimensions != NULL)
|
||||
*dimensions = metap->dimensions;
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
|
||||
/*
|
||||
* Update the start or insert page of a list
|
||||
*/
|
||||
|
||||
@@ -107,6 +107,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
|
||||
{
|
||||
/* Delete tuples */
|
||||
PageIndexMultiDelete(page, deletable, ndeletable);
|
||||
MarkBufferDirty(buf);
|
||||
GenericXLogFinish(state);
|
||||
}
|
||||
else
|
||||
|
||||
294
src/tinyint.c
294
src/tinyint.c
@@ -1,294 +0,0 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "fmgr.h"
|
||||
#include "lib/stringinfo.h"
|
||||
#include "libpq/pqformat.h"
|
||||
#include "tinyint.h"
|
||||
#include "utils/array.h"
|
||||
#include "utils/builtins.h"
|
||||
#include "utils/numeric.h"
|
||||
|
||||
/*
|
||||
* Check if array is a vector
|
||||
*/
|
||||
static bool
|
||||
ArrayIsVector(ArrayType *a)
|
||||
{
|
||||
return ARR_NDIM(a) == 1 && !array_contains_nulls(a);
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if dimensions are the same
|
||||
*/
|
||||
static int
|
||||
CheckDims(ArrayType *a, ArrayType *b)
|
||||
{
|
||||
int dima;
|
||||
int dimb;
|
||||
|
||||
if (!ArrayIsVector(a) || !ArrayIsVector(b))
|
||||
return 0;
|
||||
|
||||
dima = ARR_DIMS(a)[0];
|
||||
dimb = ARR_DIMS(b)[0];
|
||||
|
||||
if (dima != dimb)
|
||||
return 0;
|
||||
|
||||
return dima;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check range
|
||||
*/
|
||||
static void
|
||||
CheckRange(long i)
|
||||
{
|
||||
if (i < INT8_MIN || i > INT8_MAX)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("value \"%ld\" is out of range for type tinyint", i)));
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert textual representation to internal representation
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_in);
|
||||
Datum
|
||||
tinyint_in(PG_FUNCTION_ARGS)
|
||||
{
|
||||
char *s = PG_GETARG_CSTRING(0);
|
||||
const char *ptr = s;
|
||||
long i;
|
||||
char *end;
|
||||
|
||||
/* skip leading spaces */
|
||||
while (*ptr != '\0' && isspace((unsigned char) *ptr))
|
||||
ptr++;
|
||||
|
||||
if (*ptr == '\0')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("invalid input syntax for type tinyint: \"%s\"", s)));
|
||||
|
||||
i = strtol(ptr, &end, 10);
|
||||
ptr = end;
|
||||
|
||||
if (i < INT8_MIN || i > INT8_MAX)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
|
||||
errmsg("value \"%s\" is out of range for type tinyint", s)));
|
||||
|
||||
/* allow trailing whitespace, but not other trailing chars */
|
||||
while (*ptr != '\0' && isspace((unsigned char) *ptr))
|
||||
ptr++;
|
||||
|
||||
if (*ptr != '\0')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("invalid input syntax for type tinyint: \"%s\"", s)));
|
||||
|
||||
PG_RETURN_INT8(i);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert internal representation to textual representation
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_out);
|
||||
Datum
|
||||
tinyint_out(PG_FUNCTION_ARGS)
|
||||
{
|
||||
int8 num = PG_GETARG_INT8(0);
|
||||
char *result = (char *) palloc(5); /* sign, 3 digits, '\0' */
|
||||
|
||||
pg_ltoa((int32) num, result);
|
||||
PG_RETURN_CSTRING(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert external binary representation to internal representation
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_recv);
|
||||
Datum
|
||||
tinyint_recv(PG_FUNCTION_ARGS)
|
||||
{
|
||||
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
|
||||
|
||||
PG_RETURN_INT8((int8) pq_getmsgint(buf, sizeof(int8)));
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert internal representation to the external binary representation
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_send);
|
||||
Datum
|
||||
tinyint_send(PG_FUNCTION_ARGS)
|
||||
{
|
||||
int8 arg1 = PG_GETARG_INT8(0);
|
||||
StringInfoData buf;
|
||||
|
||||
pq_begintypsend(&buf);
|
||||
pq_sendint8(&buf, arg1);
|
||||
PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert integer to tinyint
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(integer_to_tinyint);
|
||||
Datum
|
||||
integer_to_tinyint(PG_FUNCTION_ARGS)
|
||||
{
|
||||
int32 i = PG_GETARG_INT32(0);
|
||||
|
||||
CheckRange(i);
|
||||
|
||||
PG_RETURN_INT8(i);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert numeric to tinyint
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(numeric_to_tinyint);
|
||||
Datum
|
||||
numeric_to_tinyint(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Numeric num = PG_GETARG_NUMERIC(0);
|
||||
int32 i = numeric_int4_opt_error(num, NULL);
|
||||
|
||||
CheckRange(i);
|
||||
|
||||
PG_RETURN_INT8(i);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the L2 distance between tinyint arrays
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_l2_distance);
|
||||
Datum
|
||||
tinyint_l2_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
ArrayType *a = PG_GETARG_ARRAYTYPE_P(0);
|
||||
ArrayType *b = PG_GETARG_ARRAYTYPE_P(1);
|
||||
int8 *ax = (int8 *) ARR_DATA_PTR(a);
|
||||
int8 *bx = (int8 *) ARR_DATA_PTR(b);
|
||||
double distance = 0.0;
|
||||
int dim = CheckDims(a, b);
|
||||
|
||||
/* TODO Decide on error or NULL */
|
||||
if (!dim)
|
||||
PG_RETURN_NULL();
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
double diff = ax[i] - bx[i];
|
||||
|
||||
distance += diff * diff;
|
||||
}
|
||||
|
||||
PG_RETURN_FLOAT8(sqrt(distance));
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the inner product of two tinyint arrays
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_inner_product);
|
||||
Datum
|
||||
tinyint_inner_product(PG_FUNCTION_ARGS)
|
||||
{
|
||||
ArrayType *a = PG_GETARG_ARRAYTYPE_P(0);
|
||||
ArrayType *b = PG_GETARG_ARRAYTYPE_P(1);
|
||||
int8 *ax = (int8 *) ARR_DATA_PTR(a);
|
||||
int8 *bx = (int8 *) ARR_DATA_PTR(b);
|
||||
double distance = 0.0;
|
||||
int dim = CheckDims(a, b);
|
||||
|
||||
/* TODO Decide on error or NULL */
|
||||
if (!dim)
|
||||
PG_RETURN_NULL();
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < dim; i++)
|
||||
distance += ax[i] * bx[i];
|
||||
|
||||
PG_RETURN_FLOAT8(distance);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the negative inner product of two tinyint arrays
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_negative_inner_product);
|
||||
Datum
|
||||
tinyint_negative_inner_product(PG_FUNCTION_ARGS)
|
||||
{
|
||||
ArrayType *a = PG_GETARG_ARRAYTYPE_P(0);
|
||||
ArrayType *b = PG_GETARG_ARRAYTYPE_P(1);
|
||||
int8 *ax = (int8 *) ARR_DATA_PTR(a);
|
||||
int8 *bx = (int8 *) ARR_DATA_PTR(b);
|
||||
double distance = 0.0;
|
||||
int dim = CheckDims(a, b);
|
||||
|
||||
/* TODO Decide on error or NULL */
|
||||
if (!dim)
|
||||
PG_RETURN_NULL();
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < dim; i++)
|
||||
distance += ax[i] * bx[i];
|
||||
|
||||
PG_RETURN_FLOAT8(distance * -1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the cosine distance between two float2 arrays
|
||||
*/
|
||||
PGDLLEXPORT PG_FUNCTION_INFO_V1(tinyint_cosine_distance);
|
||||
Datum
|
||||
tinyint_cosine_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
ArrayType *a = PG_GETARG_ARRAYTYPE_P(0);
|
||||
ArrayType *b = PG_GETARG_ARRAYTYPE_P(1);
|
||||
int8 *ax = (int8 *) ARR_DATA_PTR(a);
|
||||
int8 *bx = (int8 *) ARR_DATA_PTR(b);
|
||||
double distance = 0.0;
|
||||
double norma = 0.0;
|
||||
double normb = 0.0;
|
||||
double similarity;
|
||||
int dim = CheckDims(a, b);
|
||||
|
||||
/* TODO Decide on error or NULL */
|
||||
if (!dim)
|
||||
PG_RETURN_NULL();
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
float axi = ax[i];
|
||||
float bxi = bx[i];
|
||||
|
||||
distance += axi * bxi;
|
||||
norma += axi * axi;
|
||||
normb += bxi * bxi;
|
||||
}
|
||||
|
||||
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
|
||||
similarity = distance / sqrt(norma * normb);
|
||||
|
||||
#ifdef _MSC_VER
|
||||
/* /fp:fast may not propagate NaN */
|
||||
if (isnan(similarity))
|
||||
PG_RETURN_FLOAT8(NAN);
|
||||
#endif
|
||||
|
||||
/* Keep in range */
|
||||
if (similarity > 1)
|
||||
similarity = 1;
|
||||
else if (similarity < -1)
|
||||
similarity = -1;
|
||||
|
||||
PG_RETURN_FLOAT8(1 - similarity);
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
#ifndef TINYINT_H
|
||||
#define TINYINT_H
|
||||
|
||||
#define DatumGetInt8(X) ((int8) (X))
|
||||
#define PG_GETARG_INT8(n) DatumGetInt8(PG_GETARG_DATUM(n))
|
||||
#define PG_RETURN_INT8(x) return Int8GetDatum(x)
|
||||
|
||||
#endif
|
||||
@@ -695,8 +695,6 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||
float *ax = a->x;
|
||||
float *bx = b->x;
|
||||
float dp = 0.0;
|
||||
double distance;
|
||||
|
||||
@@ -704,7 +702,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||
|
||||
/* Auto-vectorized */
|
||||
for (int i = 0; i < a->dim; i++)
|
||||
dp += ax[i] * bx[i];
|
||||
dp += a->x[i] * b->x[i];
|
||||
|
||||
distance = (double) dp;
|
||||
|
||||
|
||||
@@ -54,85 +54,79 @@ SELECT vector_norm('[3e37,4e37]')::real;
|
||||
5e+37
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[0,0]'::vector, '[3,4]');
|
||||
SELECT l2_distance('[0,0]', '[3,4]');
|
||||
l2_distance
|
||||
-------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[0,0]'::vector, '[0,1]');
|
||||
SELECT l2_distance('[0,0]', '[0,1]');
|
||||
l2_distance
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('[1,2]'::vector, '[3]');
|
||||
SELECT l2_distance('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
|
||||
SELECT l2_distance('[3e38]', '[-3e38]');
|
||||
l2_distance
|
||||
-------------
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]'::vector, '[3,4]');
|
||||
SELECT inner_product('[1,2]', '[3,4]');
|
||||
inner_product
|
||||
---------------
|
||||
11
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('[1,2]'::vector, '[3]');
|
||||
SELECT inner_product('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT inner_product('[3e38]'::vector, '[3e38]');
|
||||
SELECT inner_product('[3e38]', '[3e38]');
|
||||
inner_product
|
||||
---------------
|
||||
Infinity
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
|
||||
SELECT cosine_distance('[1,2]', '[2,4]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
|
||||
SELECT cosine_distance('[1,2]', '[0,0]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
|
||||
SELECT cosine_distance('[1,1]', '[1,1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
|
||||
SELECT cosine_distance('[1,1]', '[-1,-1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,2]'::vector, '[3]');
|
||||
SELECT cosine_distance('[1,2]', '[3]');
|
||||
ERROR: different vector dimensions 2 and 1
|
||||
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
|
||||
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
|
||||
SELECT cosine_distance('[3e38]', '[3e38]');
|
||||
cosine_distance
|
||||
-----------------
|
||||
NaN
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
SELECT '127'::tinyint;
|
||||
tinyint
|
||||
---------
|
||||
127
|
||||
(1 row)
|
||||
|
||||
SELECT '128'::tinyint;
|
||||
ERROR: value "128" is out of range for type tinyint
|
||||
LINE 1: SELECT '128'::tinyint;
|
||||
^
|
||||
SELECT '-128'::tinyint;
|
||||
tinyint
|
||||
---------
|
||||
-128
|
||||
(1 row)
|
||||
|
||||
SELECT '-129'::tinyint;
|
||||
ERROR: value "-129" is out of range for type tinyint
|
||||
LINE 1: SELECT '-129'::tinyint;
|
||||
^
|
||||
SELECT ''::tinyint;
|
||||
ERROR: invalid input syntax for type tinyint: ""
|
||||
LINE 1: SELECT ''::tinyint;
|
||||
^
|
||||
SELECT ' 1'::tinyint;
|
||||
tinyint
|
||||
---------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT '1 '::tinyint;
|
||||
tinyint
|
||||
---------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT '1a'::tinyint;
|
||||
ERROR: invalid input syntax for type tinyint: "1a"
|
||||
LINE 1: SELECT '1a'::tinyint;
|
||||
^
|
||||
SELECT '{1,2,3}'::tinyint[];
|
||||
tinyint
|
||||
---------
|
||||
{1,2,3}
|
||||
(1 row)
|
||||
|
||||
SELECT '128'::numeric::tinyint;
|
||||
ERROR: value "128" is out of range for type tinyint
|
||||
SELECT 'NaN'::numeric::tinyint;
|
||||
ERROR: cannot convert NaN to integer
|
||||
SELECT l2_distance('{0,0}'::tinyint[], '{3,4}'::tinyint[]);
|
||||
l2_distance
|
||||
-------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('{0,0}'::tinyint[], '{0,1}'::tinyint[]);
|
||||
l2_distance
|
||||
-------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('{1,2}'::tinyint[], '{3}'::tinyint[]);
|
||||
l2_distance
|
||||
-------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT l2_distance('{3e38}'::tinyint[], '{-3e38}'::tinyint[]);
|
||||
ERROR: invalid input syntax for type tinyint: "3e38"
|
||||
LINE 1: SELECT l2_distance('{3e38}'::tinyint[], '{-3e38}'::tinyint[]...
|
||||
^
|
||||
SELECT '{0,0}'::tinyint[] <-> '{3,4}'::tinyint[];
|
||||
?column?
|
||||
----------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('{1,2}'::tinyint[], '{3,4}'::tinyint[]);
|
||||
inner_product
|
||||
---------------
|
||||
11
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('{1,2}'::tinyint[], '{3}'::tinyint[]);
|
||||
inner_product
|
||||
---------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT inner_product('{127}'::tinyint[], '{127}'::tinyint[]);
|
||||
inner_product
|
||||
---------------
|
||||
16129
|
||||
(1 row)
|
||||
|
||||
SELECT '{1,2}'::tinyint[] <#> '{3,4}'::tinyint[];
|
||||
?column?
|
||||
----------
|
||||
-11
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{1,2}'::tinyint[], '{2,4}'::tinyint[]);
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{1,2}'::tinyint[], '{0,0}'::tinyint[]);
|
||||
cosine_distance
|
||||
-----------------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{1,1}'::tinyint[], '{1,1}'::tinyint[]);
|
||||
cosine_distance
|
||||
-----------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{1,0}'::tinyint[], '{0,2}'::tinyint[]);
|
||||
cosine_distance
|
||||
-----------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{1,1}'::tinyint[], '{-1,-1}'::tinyint[]);
|
||||
cosine_distance
|
||||
-----------------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{1,2}'::tinyint[], '{3}'::tinyint[]);
|
||||
cosine_distance
|
||||
-----------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cosine_distance('{3e38}'::tinyint[], '{3e38}'::tinyint[]);
|
||||
ERROR: invalid input syntax for type tinyint: "3e38"
|
||||
LINE 1: SELECT cosine_distance('{3e38}'::tinyint[], '{3e38}'::tinyin...
|
||||
^
|
||||
SELECT '{1,2}'::tinyint[] <=> '{2,4}'::tinyint[];
|
||||
?column?
|
||||
----------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
@@ -13,24 +13,23 @@ SELECT vector_norm('[3,4]');
|
||||
SELECT vector_norm('[0,1]');
|
||||
SELECT vector_norm('[3e37,4e37]')::real;
|
||||
|
||||
SELECT l2_distance('[0,0]'::vector, '[3,4]');
|
||||
SELECT l2_distance('[0,0]'::vector, '[0,1]');
|
||||
SELECT l2_distance('[1,2]'::vector, '[3]');
|
||||
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
|
||||
SELECT l2_distance('[0,0]', '[3,4]');
|
||||
SELECT l2_distance('[0,0]', '[0,1]');
|
||||
SELECT l2_distance('[1,2]', '[3]');
|
||||
SELECT l2_distance('[3e38]', '[-3e38]');
|
||||
|
||||
SELECT inner_product('[1,2]'::vector, '[3,4]');
|
||||
SELECT inner_product('[1,2]'::vector, '[3]');
|
||||
SELECT inner_product('[3e38]'::vector, '[3e38]');
|
||||
SELECT inner_product('[1,2]', '[3,4]');
|
||||
SELECT inner_product('[1,2]', '[3]');
|
||||
SELECT inner_product('[3e38]', '[3e38]');
|
||||
|
||||
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
|
||||
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
|
||||
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
|
||||
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
|
||||
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
|
||||
SELECT cosine_distance('[1,2]'::vector, '[3]');
|
||||
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
|
||||
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
|
||||
SELECT cosine_distance('[1,2]', '[2,4]');
|
||||
SELECT cosine_distance('[1,2]', '[0,0]');
|
||||
SELECT cosine_distance('[1,1]', '[1,1]');
|
||||
SELECT cosine_distance('[1,1]', '[-1,-1]');
|
||||
SELECT cosine_distance('[1,2]', '[3]');
|
||||
SELECT cosine_distance('[1,1]', '[1.1,1.1]');
|
||||
SELECT cosine_distance('[1,1]', '[-1.1,-1.1]');
|
||||
SELECT cosine_distance('[3e38]', '[3e38]');
|
||||
|
||||
SELECT l1_distance('[0,0]', '[3,4]');
|
||||
SELECT l1_distance('[0,0]', '[0,1]');
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
SELECT '127'::tinyint;
|
||||
SELECT '128'::tinyint;
|
||||
SELECT '-128'::tinyint;
|
||||
SELECT '-129'::tinyint;
|
||||
|
||||
SELECT ''::tinyint;
|
||||
SELECT ' 1'::tinyint;
|
||||
SELECT '1 '::tinyint;
|
||||
SELECT '1a'::tinyint;
|
||||
|
||||
SELECT '{1,2,3}'::tinyint[];
|
||||
|
||||
SELECT '128'::numeric::tinyint;
|
||||
SELECT 'NaN'::numeric::tinyint;
|
||||
|
||||
SELECT l2_distance('{0,0}'::tinyint[], '{3,4}'::tinyint[]);
|
||||
SELECT l2_distance('{0,0}'::tinyint[], '{0,1}'::tinyint[]);
|
||||
SELECT l2_distance('{1,2}'::tinyint[], '{3}'::tinyint[]);
|
||||
SELECT l2_distance('{3e38}'::tinyint[], '{-3e38}'::tinyint[]);
|
||||
SELECT '{0,0}'::tinyint[] <-> '{3,4}'::tinyint[];
|
||||
|
||||
SELECT inner_product('{1,2}'::tinyint[], '{3,4}'::tinyint[]);
|
||||
SELECT inner_product('{1,2}'::tinyint[], '{3}'::tinyint[]);
|
||||
SELECT inner_product('{127}'::tinyint[], '{127}'::tinyint[]);
|
||||
SELECT '{1,2}'::tinyint[] <#> '{3,4}'::tinyint[];
|
||||
|
||||
SELECT cosine_distance('{1,2}'::tinyint[], '{2,4}'::tinyint[]);
|
||||
SELECT cosine_distance('{1,2}'::tinyint[], '{0,0}'::tinyint[]);
|
||||
SELECT cosine_distance('{1,1}'::tinyint[], '{1,1}'::tinyint[]);
|
||||
SELECT cosine_distance('{1,0}'::tinyint[], '{0,2}'::tinyint[]);
|
||||
SELECT cosine_distance('{1,1}'::tinyint[], '{-1,-1}'::tinyint[]);
|
||||
SELECT cosine_distance('{1,2}'::tinyint[], '{3}'::tinyint[]);
|
||||
SELECT cosine_distance('{3e38}'::tinyint[], '{3e38}'::tinyint[]);
|
||||
SELECT '{1,2}'::tinyint[] <=> '{2,4}'::tinyint[];
|
||||
@@ -19,6 +19,8 @@ sub test_index_replay
|
||||
|
||||
# Wait for replica to catch up
|
||||
my $applname = $node_replica->name;
|
||||
|
||||
my $server_version_num = $node_primary->safe_psql("postgres", "SHOW server_version_num");
|
||||
my $caughtup_query = "SELECT pg_current_wal_lsn() <= replay_lsn FROM pg_stat_replication WHERE application_name = '$applname';";
|
||||
$node_primary->poll_query_until('postgres', $caughtup_query)
|
||||
or die "Timed out while waiting for replica 1 to catch up";
|
||||
|
||||
@@ -94,7 +94,7 @@ for my $i (0 .. $#operators)
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
# TODO fix test
|
||||
test_recall(1, 0.71, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
@@ -115,7 +115,7 @@ for my $i (0 .. $#operators)
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
# TODO fix test
|
||||
test_recall(1, 0.71, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
@@ -19,6 +19,8 @@ sub test_index_replay
|
||||
|
||||
# Wait for replica to catch up
|
||||
my $applname = $node_replica->name;
|
||||
|
||||
my $server_version_num = $node_primary->safe_psql("postgres", "SHOW server_version_num");
|
||||
my $caughtup_query = "SELECT pg_current_wal_lsn() <= replay_lsn FROM pg_stat_replication WHERE application_name = '$applname';";
|
||||
$node_primary->poll_query_until('postgres', $caughtup_query)
|
||||
or die "Timed out while waiting for replica 1 to catch up";
|
||||
|
||||
@@ -89,7 +89,7 @@ foreach (@queries)
|
||||
test_recall(0.20, $limit, "before vacuum");
|
||||
test_recall(0.95, 100, "before vacuum");
|
||||
|
||||
# TODO Test concurrent inserts with vacuum
|
||||
# TODO test concurrent inserts with vacuum
|
||||
$node->safe_psql("postgres", "VACUUM tst;");
|
||||
|
||||
test_recall(0.95, $limit, "after vacuum");
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
my @expected;
|
||||
my $limit = 20;
|
||||
|
||||
sub test_recall
|
||||
{
|
||||
my ($probes, $min, $operator) = @_;
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan using idx on tst/);
|
||||
|
||||
for my $i (0 .. $#queries)
|
||||
{
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
|
||||
));
|
||||
my @actual_ids = split("\n", $actual);
|
||||
my %actual_set = map { $_ => 1 } @actual_ids;
|
||||
|
||||
my @expected_ids = split("\n", $expected[$i]);
|
||||
|
||||
foreach (@expected_ids)
|
||||
{
|
||||
if (exists($actual_set{$_}))
|
||||
{
|
||||
$correct++;
|
||||
}
|
||||
$total++;
|
||||
}
|
||||
}
|
||||
|
||||
cmp_ok($correct / $total, ">=", $min, $operator);
|
||||
}
|
||||
|
||||
# Initialize node
|
||||
$node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector(3));");
|
||||
|
||||
# Generate queries
|
||||
for (1 .. 20)
|
||||
{
|
||||
my $r1 = rand();
|
||||
my $r2 = rand();
|
||||
my $r3 = rand();
|
||||
push(@queries, "[$r1,$r2,$r3]");
|
||||
}
|
||||
|
||||
# Check each index type
|
||||
my @operators = ("<->", "<#>", "<=>");
|
||||
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
|
||||
|
||||
for my $i (0 .. $#operators)
|
||||
{
|
||||
my $operator = $operators[$i];
|
||||
my $opclass = $opclasses[$i];
|
||||
|
||||
# Add index
|
||||
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v $opclass);");
|
||||
|
||||
# Use concurrent inserts
|
||||
$node->pgbench(
|
||||
"--no-vacuum --client=10 --transactions=1000",
|
||||
0,
|
||||
[qr{actually processed}],
|
||||
[qr{^$}],
|
||||
"concurrent INSERTs",
|
||||
{
|
||||
"017_ivfflat_insert_recall_$opclass" => "INSERT INTO tst (v) SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 10) i;"
|
||||
}
|
||||
);
|
||||
|
||||
# Get exact results
|
||||
@expected = ();
|
||||
foreach (@queries)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_indexscan = off;
|
||||
SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;
|
||||
));
|
||||
push(@expected, $res);
|
||||
}
|
||||
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>")
|
||||
{
|
||||
# TODO Fix test (uniform random vectors all have similar inner product)
|
||||
test_recall(1, 0.71, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
# Account for equal distances
|
||||
test_recall(100, 0.9925, $operator);
|
||||
|
||||
$node->safe_psql("postgres", "DROP INDEX idx;");
|
||||
$node->safe_psql("postgres", "TRUNCATE tst;");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,43 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
|
||||
my $dim = 3;
|
||||
|
||||
my $array_sql = join(",", ('random()') x $dim);
|
||||
|
||||
# Initialize node
|
||||
my $node = get_new_node('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table and index
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
|
||||
);
|
||||
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
|
||||
|
||||
# Delete data
|
||||
$node->safe_psql("postgres", "DELETE FROM tst WHERE i % 100 != 0;");
|
||||
|
||||
my $exp = $node->safe_psql("postgres", qq(
|
||||
SET enable_indexscan = off;
|
||||
SELECT i FROM tst ORDER BY v <-> '[0,0,0]';
|
||||
));
|
||||
|
||||
# Run twice to make sure correct tuples marked as dead
|
||||
for (1 .. 2)
|
||||
{
|
||||
my $res = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = 100;
|
||||
SELECT i FROM tst ORDER BY v <-> '[0,0,0]';
|
||||
));
|
||||
is($res, $exp);
|
||||
}
|
||||
|
||||
done_testing();
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat and hnsw access methods'
|
||||
default_version = '0.5.0'
|
||||
comment = 'vector data type and ivfflat access method'
|
||||
default_version = '0.4.4'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user