Compare commits

..

2 Commits

Author SHA1 Message Date
Andrew Kane
0ff9f6511a Use f32 [skip ci] 2024-04-27 23:02:00 -07:00
Andrew Kane
17855c9861 Started Neon intrinsics [skip ci] 2024-04-27 22:50:47 -07:00
87 changed files with 1155 additions and 2863 deletions

View File

@@ -8,18 +8,18 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
# - postgres: 18
# os: ubuntu-24.04
- postgres: 17 - postgres: 17
os: ubuntu-24.04 os: ubuntu-22.04
- postgres: 16 - postgres: 16
os: ubuntu-22.04 os: ubuntu-22.04
- postgres: 15 - postgres: 15
os: ubuntu-22.04 os: ubuntu-22.04
- postgres: 14 - postgres: 14
os: ubuntu-20.04 os: ubuntu-22.04
- postgres: 13 - postgres: 13
os: ubuntu-20.04 os: ubuntu-20.04
- postgres: 12
os: ubuntu-20.04
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1

View File

@@ -1,35 +1,4 @@
## 0.8.0 (unreleased) ## 0.7.0 (unreleased)
- Added support for inline filtering with HNSW
- Added casts for arrays to `sparsevec`
- Improved cost estimation
- Improved performance of HNSW inserts and on-disk index builds
- Reduced memory usage for HNSW index scans
- Dropped support for Postgres 12
## 0.7.4 (2024-08-05)
- Fixed locking for parallel HNSW index builds
- Fixed compilation error with GCC 14 on i386 when SSE2 is not enabled
## 0.7.3 (2024-07-22)
- Fixed `failed to add index item` error with `sparsevec`
- Fixed compilation error with FreeBSD ARM
- Fixed compilation warning with MSVC and Postgres 16
## 0.7.2 (2024-06-11)
- Fixed initialization fork for indexes on unlogged tables
## 0.7.1 (2024-06-03)
- Improved performance of on-disk HNSW index builds
- Fixed `undefined symbol` error with GCC 8
- Fixed compilation error with universal binaries on Mac
- Fixed compilation warning with Clang < 14
## 0.7.0 (2024-04-29)
- Added `halfvec` type - Added `halfvec` type
- Added `sparsevec` type - Added `sparsevec` type

View File

@@ -1,4 +1,4 @@
ARG PG_MAJOR=17 ARG PG_MAJOR=16
FROM postgres:$PG_MAJOR FROM postgres:$PG_MAJOR
ARG PG_MAJOR ARG PG_MAJOR

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.7.4", "version": "0.6.2",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.7.4", "version": "0.6.2",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,9 +1,8 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.7.4 EXTVERSION = 0.6.2
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql) DATA = $(wildcard sql/*--*.sql)
DATA_built = sql/$(EXTENSION)--$(EXTVERSION).sql
OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
@@ -43,6 +42,8 @@ all: sql/$(EXTENSION)--$(EXTVERSION).sql
sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql
cp $< $@ cp $< $@
EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql
PG_CONFIG ?= pg_config PG_CONFIG ?= pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs) PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS) include $(PGXS)
@@ -52,7 +53,7 @@ ifeq ($(PROVE),)
PROVE = prove PROVE = prove
endif endif
# for Postgres < 15 # for Postgres 15
PROVE_FLAGS += -I ./test/perl PROVE_FLAGS += -I ./test/perl
prove_installcheck: prove_installcheck:
@@ -66,7 +67,7 @@ dist:
git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master
# for Docker # for Docker
PG_MAJOR ?= 17 PG_MAJOR ?= 16
.PHONY: docker .PHONY: docker

View File

@@ -1,11 +1,10 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.7.4 EXTVERSION = 0.6.2
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION) REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# For /arch flags # For /arch flags
@@ -20,6 +19,11 @@ PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
# https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages # https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
# PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2 # PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
all: sql\$(EXTENSION)--$(EXTVERSION).sql
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
# TODO use pg_config # TODO use pg_config
!ifndef PGROOT !ifndef PGROOT
!error PGROOT is not set !error PGROOT is not set
@@ -39,18 +43,15 @@ SHLIB = $(EXTENSION).dll
LIBS = "$(LIBDIR)\postgres.lib" LIBS = "$(LIBDIR)\postgres.lib"
all: $(SHLIB) $(DATA_built)
.c.obj: .c.obj:
$(CC) $(CFLAGS) /c $< /Fo$@ $(CC) $(CFLAGS) /c $< /Fo$@
$(SHLIB): $(OBJS) $(SHLIB): $(OBJS)
$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB) $(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql all: $(SHLIB)
copy sql\$(EXTENSION).sql $@
install: all install:
copy $(SHLIB) "$(PKGLIBDIR)" copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension" copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension" copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
@@ -69,6 +70,6 @@ uninstall:
clean: clean:
del /f $(SHLIB) $(EXTENSION).lib $(EXTENSION).exp del /f $(SHLIB) $(EXTENSION).lib $(EXTENSION).exp
del /f $(DATA_built)
del /f $(OBJS) del /f $(OBJS)
del /f sql\$(EXTENSION)--$(EXTVERSION).sql
del /f /s /q results regression.diffs regression.out tmp_check tmp_check_iso log output_iso del /f /s /q results regression.diffs regression.out tmp_check tmp_check_iso log output_iso

202
README.md
View File

@@ -5,8 +5,7 @@ Open-source vector similarity search for Postgres
Store your vectors with the rest of your data. Supports: Store your vectors with the rest of your data. Supports:
- exact and approximate nearest neighbor search - exact and approximate nearest neighbor search
- single-precision, half-precision, binary, and sparse vectors - L2 distance, inner product, and cosine distance
- L2 distance, inner product, cosine distance, L1 distance, Hamming distance, and Jaccard distance
- any [language](#languages) with a Postgres client - any [language](#languages) with a Postgres client
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
@@ -21,7 +20,7 @@ Compile and install the extension (supports Postgres 12+)
```sh ```sh
cd /tmp cd /tmp
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -46,14 +45,12 @@ Then use `nmake` to build:
```cmd ```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16" set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP% cd %TEMP%
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
nmake /F Makefile.win nmake /F Makefile.win
nmake /F Makefile.win install nmake /F Makefile.win install
``` ```
Note: Postgres 17 is not supported yet due to an upstream issue
See the [installation notes](#installation-notes---windows) if you run into issues See the [installation notes](#installation-notes---windows) if you run into issues
You can also install it with [Docker](#docker) or [conda-forge](#conda-forge). You can also install it with [Docker](#docker) or [conda-forge](#conda-forge).
@@ -84,7 +81,7 @@ Get the nearest neighbors by L2 distance
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
``` ```
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0) Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, unreleased)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
@@ -102,15 +99,13 @@ Or add a vector column to an existing table
ALTER TABLE items ADD COLUMN embedding vector(3); ALTER TABLE items ADD COLUMN embedding vector(3);
``` ```
Also supports [half-precision](#half-precision-vectors), [binary](#binary-vectors), and [sparse](#sparse-vectors) vectors
Insert vectors Insert vectors
```sql ```sql
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]'); INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
``` ```
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)) Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py))
```sql ```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY); COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -148,9 +143,7 @@ Supported distance functions are:
- `<->` - L2 distance - `<->` - L2 distance
- `<#>` - (negative) inner product - `<#>` - (negative) inner product
- `<=>` - cosine distance - `<=>` - cosine distance
- `<+>` - L1 distance (added in 0.7.0) - `<+>` - L1 distance (unreleased)
- `<~>` - Hamming distance (binary vectors, added in 0.7.0)
- `<%>` - Jaccard distance (binary vectors, added in 0.7.0)
Get the nearest neighbors to a row Get the nearest neighbors to a row
@@ -208,7 +201,7 @@ You can add an index to use approximate nearest neighbor search, which trades so
Supported index types are: Supported index types are:
- [HNSW](#hnsw) - [HNSW](#hnsw) - added in 0.5.0
- [IVFFlat](#ivfflat) - [IVFFlat](#ivfflat)
## HNSW ## HNSW
@@ -223,8 +216,6 @@ L2 distance
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops); CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
``` ```
Note: Use `halfvec_l2_ops` for `halfvec` and `sparsevec_l2_ops` for `sparsevec` (and similar with the other distance functions)
Inner product Inner product
```sql ```sql
@@ -237,19 +228,19 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops); CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
``` ```
L1 distance - added in 0.7.0 L1 distance - unreleased
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops); CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
``` ```
Hamming distance - added in 0.7.0 Hamming distance - unreleased
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops); CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
``` ```
Jaccard distance - added in 0.7.0 Jaccard distance - unreleased
```sql ```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops); CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
@@ -258,9 +249,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
Supported types are: Supported types are:
- `vector` - up to 2,000 dimensions - `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions (added in 0.7.0) - `halfvec` - up to 4,000 dimensions (unreleased)
- `bit` - up to 64,000 dimensions (added in 0.7.0) - `bit` - up to 64,000 dimensions (unreleased)
- `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0) - `sparsevec` - up to 1,000 non-zero elements (unreleased)
### Index Options ### Index Options
@@ -353,8 +344,6 @@ L2 distance
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
``` ```
Note: Use `halfvec_l2_ops` for `halfvec` (and similar with the other distance functions)
Inner product Inner product
```sql ```sql
@@ -367,7 +356,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
``` ```
Hamming distance - added in 0.7.0 Hamming distance - unreleased
```sql ```sql
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100); CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
@@ -376,8 +365,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10
Supported types are: Supported types are:
- `vector` - up to 2,000 dimensions - `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions (added in 0.7.0) - `halfvec` - up to 4,000 dimensions (unreleased)
- `bit` - up to 64,000 dimensions (added in 0.7.0) - `bit` - up to 64,000 dimensions (unreleased)
### Query Options ### Query Options
@@ -439,12 +428,6 @@ Create an index on one [or more](https://www.postgresql.org/docs/current/indexes
CREATE INDEX ON items (category_id); CREATE INDEX ON items (category_id);
``` ```
Or a composite HNSW index for approximate search (added in 0.8.0)
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops, category_id);
```
Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
```sql ```sql
@@ -457,9 +440,9 @@ Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id); CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
``` ```
## Half-Precision Vectors ## Half Vectors
*Added in 0.7.0* *Unreleased*
Use the `halfvec` type to store half-precision vectors Use the `halfvec` type to store half-precision vectors
@@ -467,11 +450,11 @@ Use the `halfvec` type to store half-precision vectors
CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3)); CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
``` ```
## Half-Precision Indexing ## Half Indexing
*Added in 0.7.0* *Unreleased*
Index vectors at half precision for smaller indexes Index vectors at half precision for smaller indexes and faster build times
```sql ```sql
CREATE INDEX ON items USING hnsw ((embedding::halfvec(3)) halfvec_l2_ops); CREATE INDEX ON items USING hnsw ((embedding::halfvec(3)) halfvec_l2_ops);
@@ -485,30 +468,30 @@ SELECT * FROM items ORDER BY embedding::halfvec(3) <-> '[1,2,3]' LIMIT 5;
## Binary Vectors ## Binary Vectors
Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/imagehash/example.py)) Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py))
```sql ```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3)); CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111'); INSERT INTO items (embedding) VALUES ('000'), ('111');
``` ```
Get the nearest neighbors by Hamming distance (added in 0.7.0) Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Or (before 0.7.0)
```sql ```sql
SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5; SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5;
``` ```
Or (unreleased)
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Also supports Jaccard distance (`<%>`) Also supports Jaccard distance (`<%>`)
## Binary Quantization ## Binary Quantization
*Added in 0.7.0* *Unreleased*
Use expression indexing for binary quantization Use expression indexing for binary quantization
@@ -532,7 +515,7 @@ SELECT * FROM (
## Sparse Vectors ## Sparse Vectors
*Added in 0.7.0* *Unreleased*
Use the `sparsevec` type to store sparse vectors Use the `sparsevec` type to store sparse vectors
@@ -563,11 +546,11 @@ SELECT id, content FROM items, plainto_tsquery('hello search') query
WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5; WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
``` ```
You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/cross_encoder.py) to combine results. You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search_rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py) to combine results.
## Indexing Subvectors ## Indexing Subvectors
*Added in 0.7.0* *Unreleased*
Use expression indexing to index subvectors Use expression indexing to index subvectors
@@ -609,7 +592,7 @@ Be sure to restart Postgres for changes to take effect.
### Loading ### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)). Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py)).
```sql ```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY); COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -699,7 +682,7 @@ Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring). Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)). Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
## Languages ## Languages
@@ -743,7 +726,7 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions? #### What if I want to index vectors with more than 2,000 dimensions?
You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction). Youll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
#### Can I store vectors with different dimensions in the same column? #### Can I store vectors with different dimensions in the same column?
@@ -806,7 +789,7 @@ SELECT pg_size_pretty(pg_relation_size('index_name'));
#### Why isnt a query using an index? #### Why isnt a query using an index?
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator (not an expression) in ascending order. The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator, not an expression.
```sql ```sql
-- index -- index
@@ -881,23 +864,23 @@ Operator | Description | Added
\+ | element-wise addition | \+ | element-wise addition |
\- | element-wise subtraction | \- | element-wise subtraction |
\* | element-wise multiplication | 0.5.0 \* | element-wise multiplication | 0.5.0
\|\| | concatenate | 0.7.0 \|\| | concatenate | unreleased
<-> | Euclidean distance | <-> | Euclidean distance |
<#> | negative inner product | <#> | negative inner product |
<=> | cosine distance | <=> | cosine distance |
<+> | taxicab distance | 0.7.0 <+> | taxicab distance | unreleased
### Vector Functions ### Vector Functions
Function | Description | Added Function | Description | Added
--- | --- | --- --- | --- | ---
binary_quantize(vector) → bit | binary quantize | 0.7.0 binary_quantize(vector) → bit | binary quantize | unreleased
cosine_distance(vector, vector) → double precision | cosine distance | cosine_distance(vector, vector) → double precision | cosine distance |
inner_product(vector, vector) → double precision | inner product | inner_product(vector, vector) → double precision | inner product |
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0 l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
l2_distance(vector, vector) → double precision | Euclidean distance | l2_distance(vector, vector) → double precision | Euclidean distance |
l2_normalize(vector) → vector | Normalize with Euclidean norm | 0.7.0 l2_normalize(vector) → vector | Normalize with Euclidean norm | unreleased
subvector(vector, integer, integer) → vector | subvector | 0.7.0 subvector(vector, integer, integer) → vector | subvector | unreleased
vector_dims(vector) → integer | number of dimensions | vector_dims(vector) → integer | number of dimensions |
vector_norm(vector) → double precision | Euclidean norm | vector_norm(vector) → double precision | Euclidean norm |
@@ -916,35 +899,35 @@ Each half vector takes `2 * dimensions + 8` bytes of storage. Each element is a
Operator | Description | Added Operator | Description | Added
--- | --- | --- --- | --- | ---
\+ | element-wise addition | 0.7.0 \+ | element-wise addition | unreleased
\- | element-wise subtraction | 0.7.0 \- | element-wise subtraction | unreleased
\* | element-wise multiplication | 0.7.0 \* | element-wise multiplication | unreleased
\|\| | concatenate | 0.7.0 \|\| | concatenate | unreleased
<-> | Euclidean distance | 0.7.0 <-> | Euclidean distance | unreleased
<#> | negative inner product | 0.7.0 <#> | negative inner product | unreleased
<=> | cosine distance | 0.7.0 <=> | cosine distance | unreleased
<+> | taxicab distance | 0.7.0 <+> | taxicab distance | unreleased
### Halfvec Functions ### Halfvec Functions
Function | Description | Added Function | Description | Added
--- | --- | --- --- | --- | ---
binary_quantize(halfvec) → bit | binary quantize | 0.7.0 binary_quantize(halfvec) → bit | binary quantize | unreleased
cosine_distance(halfvec, halfvec) → double precision | cosine distance | 0.7.0 cosine_distance(halfvec, halfvec) → double precision | cosine distance | unreleased
inner_product(halfvec, halfvec) → double precision | inner product | 0.7.0 inner_product(halfvec, halfvec) → double precision | inner product | unreleased
l1_distance(halfvec, halfvec) → double precision | taxicab distance | 0.7.0 l1_distance(halfvec, halfvec) → double precision | taxicab distance | unreleased
l2_distance(halfvec, halfvec) → double precision | Euclidean distance | 0.7.0 l2_distance(halfvec, halfvec) → double precision | Euclidean distance | unreleased
l2_norm(halfvec) → double precision | Euclidean norm | 0.7.0 l2_norm(halfvec) → double precision | Euclidean norm | unreleased
l2_normalize(halfvec) → halfvec | Normalize with Euclidean norm | 0.7.0 l2_normalize(halfvec) → halfvec | Normalize with Euclidean norm | unreleased
subvector(halfvec, integer, integer) → halfvec | subvector | 0.7.0 subvector(halfvec, integer, integer) → halfvec | subvector | unreleased
vector_dims(halfvec) → integer | number of dimensions | 0.7.0 vector_dims(halfvec) → integer | number of dimensions | unreleased
### Halfvec Aggregate Functions ### Halfvec Aggregate Functions
Function | Description | Added Function | Description | Added
--- | --- | --- --- | --- | ---
avg(halfvec) → halfvec | average | 0.7.0 avg(halfvec) → halfvec | average | unreleased
sum(halfvec) → halfvec | sum | 0.7.0 sum(halfvec) → halfvec | sum | unreleased
### Bit Type ### Bit Type
@@ -954,15 +937,15 @@ Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres d
Operator | Description | Added Operator | Description | Added
--- | --- | --- --- | --- | ---
<~> | Hamming distance | 0.7.0 <~> | Hamming distance | unreleased
<%> | Jaccard distance | 0.7.0 <%> | Jaccard distance | unreleased
### Bit Functions ### Bit Functions
Function | Description | Added Function | Description | Added
--- | --- | --- --- | --- | ---
hamming_distance(bit, bit) → double precision | Hamming distance | 0.7.0 hamming_distance(bit, bit) → double precision | Hamming distance | unreleased
jaccard_distance(bit, bit) → double precision | Jaccard distance | 0.7.0 jaccard_distance(bit, bit) → double precision | Jaccard distance | unreleased
### Sparsevec Type ### Sparsevec Type
@@ -972,21 +955,21 @@ Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each ele
Operator | Description | Added Operator | Description | Added
--- | --- | --- --- | --- | ---
<-> | Euclidean distance | 0.7.0 <-> | Euclidean distance | unreleased
<#> | negative inner product | 0.7.0 <#> | negative inner product | unreleased
<=> | cosine distance | 0.7.0 <=> | cosine distance | unreleased
<+> | taxicab distance | 0.7.0 <+> | taxicab distance | unreleased
### Sparsevec Functions ### Sparsevec Functions
Function | Description | Added Function | Description | Added
--- | --- | --- --- | --- | ---
cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | 0.7.0 cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | unreleased
inner_product(sparsevec, sparsevec) → double precision | inner product | 0.7.0 inner_product(sparsevec, sparsevec) → double precision | inner product | unreleased
l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | 0.7.0 l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | unreleased
l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | 0.7.0 l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | unreleased
l2_norm(sparsevec) → double precision | Euclidean norm | 0.7.0 l2_norm(sparsevec) → double precision | Euclidean norm | unreleased
l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0 l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | unreleased
## Installation Notes - Linux and Mac ## Installation Notes - Linux and Mac
@@ -995,7 +978,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with: If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
```sh ```sh
export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config export PG_CONFIG=/Library/PostgreSQL/16/bin/pg_config
``` ```
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use: Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1006,11 +989,11 @@ sudo --preserve-env=PG_CONFIG make install
A few common paths on Mac are: A few common paths on Mac are:
- EDB installer - `/Library/PostgreSQL/17/bin/pg_config` - EDB installer - `/Library/PostgreSQL/16/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config` - Homebrew (arm64) - `/opt/homebrew/opt/postgresql@16/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config` - Homebrew (x86-64) - `/usr/local/opt/postgresql@16/bin/pg_config`
Note: Replace `17` with your Postgres server version Note: Replace `16` with your Postgres server version
### Missing Header ### Missing Header
@@ -1019,10 +1002,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
For Ubuntu and Debian, use: For Ubuntu and Debian, use:
```sh ```sh
sudo apt install postgresql-server-dev-17 sudo apt install postgresql-server-dev-16
``` ```
Note: Replace `17` with your Postgres server version Note: Replace `16` with your Postgres server version
### Missing SDK ### Missing SDK
@@ -1055,17 +1038,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with: Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
```sh ```sh
docker pull pgvector/pgvector:pg17 docker pull pgvector/pgvector:pg16
``` ```
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way). This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
You can also build the image manually: You can also build the image manually:
```sh ```sh
git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector . docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
``` ```
### Homebrew ### Homebrew
@@ -1076,7 +1059,7 @@ With Homebrew Postgres, you can use:
brew install pgvector brew install pgvector
``` ```
Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas Note: This only adds it to the `postgresql@14` formula
### PGXN ### PGXN
@@ -1091,22 +1074,22 @@ pgxn install vector
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run: Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
```sh ```sh
sudo apt install postgresql-17-pgvector sudo apt install postgresql-16-pgvector
``` ```
Note: Replace `17` with your Postgres server version Note: Replace `16` with your Postgres server version
### Yum ### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run: RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
```sh ```sh
sudo yum install pgvector_17 sudo yum install pgvector_16
# or # or
sudo dnf install pgvector_17 sudo dnf install pgvector_16
``` ```
Note: Replace `17` with your Postgres server version Note: Replace `16` with your Postgres server version
### pkg ### pkg
@@ -1195,7 +1178,6 @@ Thanks to:
- [k-means++: The Advantage of Careful Seeding](https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf) - [k-means++: The Advantage of Careful Seeding](https://theory.stanford.edu/~sergei/papers/kMeansPP-soda.pdf)
- [Concept Decompositions for Large Sparse Text Data using Clustering](https://www.cs.utexas.edu/users/inderjit/public_papers/concept_mlj.pdf) - [Concept Decompositions for Large Sparse Text Data using Clustering](https://www.cs.utexas.edu/users/inderjit/public_papers/concept_mlj.pdf)
- [Efficient and Robust Approximate Nearest Neighbor Search using Hierarchical Navigable Small World Graphs](https://arxiv.org/ftp/arxiv/papers/1603/1603.09320.pdf) - [Efficient and Robust Approximate Nearest Neighbor Search using Hierarchical Navigable Small World Graphs](https://arxiv.org/ftp/arxiv/papers/1603/1603.09320.pdf)
- [HQANN: Efficient and Robust Similarity Search for Hybrid Queries with Structured and Unstructured Constraints](https://arxiv.org/pdf/2207.07940.pdf)
## History ## History

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.1'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.2'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.4'" to load this file. \quit

View File

@@ -1,34 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.0'" to load this file. \quit
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (integer[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;
CREATE FUNCTION hnsw_attribute_distance(integer, integer) RETURNS float8
AS 'MODULE_PATHNAME', 'hnsw_int4_attribute_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR CLASS vector_integer_ops
DEFAULT FOR TYPE integer USING hnsw AS
OPERATOR 2 = (integer, integer),
FUNCTION 4 hnsw_attribute_distance(integer, integer);

View File

@@ -782,18 +782,6 @@ CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparseve
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec casts -- sparsevec casts
CREATE CAST (sparsevec AS sparsevec) CREATE CAST (sparsevec AS sparsevec)
@@ -811,18 +799,6 @@ CREATE CAST (sparsevec AS halfvec)
CREATE CAST (halfvec AS sparsevec) CREATE CAST (halfvec AS sparsevec)
WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT; WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (integer[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;
-- sparsevec operators -- sparsevec operators
CREATE OPERATOR <-> ( CREATE OPERATOR <-> (
@@ -916,13 +892,3 @@ CREATE OPERATOR CLASS sparsevec_l1_ops
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops, OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(sparsevec, sparsevec), FUNCTION 1 l1_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal); FUNCTION 3 hnsw_sparsevec_support(internal);
-- hnsw attributes
CREATE FUNCTION hnsw_attribute_distance(integer, integer) RETURNS float8
AS 'MODULE_PATHNAME', 'hnsw_int4_attribute_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR CLASS vector_integer_ops
DEFAULT FOR TYPE integer USING hnsw AS
OPERATOR 2 = (integer, integer),
FUNCTION 4 hnsw_attribute_distance(integer, integer);

View File

@@ -11,7 +11,7 @@
#ifdef BIT_DISPATCH #ifdef BIT_DISPATCH
#include <immintrin.h> #include <immintrin.h>
#if defined(USE__GET_CPUID) #if defined(HAVE__GET_CPUID)
#include <cpuid.h> #include <cpuid.h>
#else #else
#include <intrin.h> #include <intrin.h>
@@ -173,7 +173,7 @@ SupportsAvx512Popcount()
{ {
unsigned int exx[4] = {0, 0, 0, 0}; unsigned int exx[4] = {0, 0, 0, 0};
#if defined(USE__GET_CPUID) #if defined(HAVE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#else #else
__cpuid(exx, 1); __cpuid(exx, 1);
@@ -187,7 +187,7 @@ SupportsAvx512Popcount()
if ((_xgetbv(0) & 0xe6) != 0xe6) if ((_xgetbv(0) & 0xe6) != 0xe6)
return false; return false;
#if defined(USE__GET_CPUID) #if defined(HAVE__GET_CPUID)
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]);
#else #else
__cpuidex(exx, 7, 0); __cpuidex(exx, 7, 0);

View File

@@ -4,8 +4,8 @@
#include "postgres.h" #include "postgres.h"
/* Check version in first header */ /* Check version in first header */
#if PG_VERSION_NUM < 130000 #if PG_VERSION_NUM < 120000
#error "Requires PostgreSQL 13+" #error "Requires PostgreSQL 12+"
#endif #endif
extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance); extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);

View File

@@ -3,7 +3,6 @@
#include "bitutils.h" #include "bitutils.h"
#include "bitvec.h" #include "bitvec.h"
#include "utils/varbit.h" #include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000 #if PG_VERSION_NUM >= 160000
#include "varatt.h" #include "varatt.h"
@@ -41,7 +40,7 @@ CheckDims(VarBit *a, VarBit *b)
/* /*
* Get the Hamming distance between two bit vectors * Get the Hamming distance between two bit vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hamming_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(hamming_distance);
Datum Datum
hamming_distance(PG_FUNCTION_ARGS) hamming_distance(PG_FUNCTION_ARGS)
{ {
@@ -56,7 +55,7 @@ hamming_distance(PG_FUNCTION_ARGS)
/* /*
* Get the Jaccard distance between two bit vectors * Get the Jaccard distance between two bit vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(jaccard_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(jaccard_distance);
Datum Datum
jaccard_distance(PG_FUNCTION_ARGS) jaccard_distance(PG_FUNCTION_ARGS)
{ {

View File

@@ -1,12 +1,14 @@
#include "postgres.h" #include "postgres.h"
#include <arm_neon.h>
#include "halfutils.h" #include "halfutils.h"
#include "halfvec.h" #include "halfvec.h"
#ifdef HALFVEC_DISPATCH #ifdef HALFVEC_DISPATCH
#include <immintrin.h> #include <immintrin.h>
#if defined(USE__GET_CPUID) #if defined(HAVE__GET_CPUID)
#include <cpuid.h> #include <cpuid.h>
#else #else
#include <intrin.h> #include <intrin.h>
@@ -28,9 +30,27 @@ static float
HalfvecL2SquaredDistanceDefault(int dim, half * ax, half * bx) HalfvecL2SquaredDistanceDefault(int dim, half * ax, half * bx)
{ {
float distance = 0.0; float distance = 0.0;
int i = 0;
/* TODO Improve */
#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
int count = (dim / 4) * 4;
float32x4_t dist = vmovq_n_f32(0);
for (; i < count; i += 4)
{
float16x4_t axs = vld1_f16((const __fp16 *) (ax + i));
float16x4_t bxs = vld1_f16((const __fp16 *) (bx + i));
float32x4_t diff = vsubq_f32(vcvt_f32_f16(axs), vcvt_f32_f16(bxs));
dist = vfmaq_f32(dist, diff, diff);
}
distance += vaddvq_f32(dist);
#endif
/* Auto-vectorized */ /* Auto-vectorized */
for (int i = 0; i < dim; i++) for (; i < dim; i++)
{ {
float diff = HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]); float diff = HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]);
@@ -254,7 +274,7 @@ SupportsCpuFeature(unsigned int feature)
{ {
unsigned int exx[4] = {0, 0, 0, 0}; unsigned int exx[4] = {0, 0, 0, 0};
#if defined(USE__GET_CPUID) #if defined(HAVE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#else #else
__cpuid(exx, 1); __cpuid(exx, 1);

View File

@@ -19,6 +19,11 @@
#include "utils/numeric.h" #include "utils/numeric.h"
#include "vector.h" #include "vector.h"
#if PG_VERSION_NUM < 130000
#define TYPALIGN_DOUBLE 'd'
#define TYPALIGN_INT 'i'
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1) #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1)) #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -159,10 +164,28 @@ CheckStateArray(ArrayType *statearray, const char *caller)
return (float8 *) ARR_DATA_PTR(statearray); return (float8 *) ARR_DATA_PTR(statearray);
} }
#if PG_VERSION_NUM < 120003
static pg_noinline void
float_overflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: overflow")));
}
static pg_noinline void
float_underflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: underflow")));
}
#endif
/* /*
* Convert textual representation to internal representation * Convert textual representation to internal representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_in);
Datum Datum
halfvec_in(PG_FUNCTION_ARGS) halfvec_in(PG_FUNCTION_ARGS)
{ {
@@ -276,7 +299,7 @@ halfvec_in(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to textual representation * Convert internal representation to textual representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_out); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_out);
Datum Datum
halfvec_out(PG_FUNCTION_ARGS) halfvec_out(PG_FUNCTION_ARGS)
{ {
@@ -322,7 +345,7 @@ halfvec_out(PG_FUNCTION_ARGS)
/* /*
* Convert type modifier * Convert type modifier
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_typmod_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_typmod_in);
Datum Datum
halfvec_typmod_in(PG_FUNCTION_ARGS) halfvec_typmod_in(PG_FUNCTION_ARGS)
{ {
@@ -353,7 +376,7 @@ halfvec_typmod_in(PG_FUNCTION_ARGS)
/* /*
* Convert external binary representation to internal representation * Convert external binary representation to internal representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_recv); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_recv);
Datum Datum
halfvec_recv(PG_FUNCTION_ARGS) halfvec_recv(PG_FUNCTION_ARGS)
{ {
@@ -387,7 +410,7 @@ halfvec_recv(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to the external binary representation * Convert internal representation to the external binary representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_send); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_send);
Datum Datum
halfvec_send(PG_FUNCTION_ARGS) halfvec_send(PG_FUNCTION_ARGS)
{ {
@@ -407,7 +430,7 @@ halfvec_send(PG_FUNCTION_ARGS)
* Convert half vector to half vector * Convert half vector to half vector
* This is needed to check the type modifier * This is needed to check the type modifier
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec);
Datum Datum
halfvec(PG_FUNCTION_ARGS) halfvec(PG_FUNCTION_ARGS)
{ {
@@ -422,7 +445,7 @@ halfvec(PG_FUNCTION_ARGS)
/* /*
* Convert array to half vector * Convert array to half vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_halfvec); PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_halfvec);
Datum Datum
array_to_halfvec(PG_FUNCTION_ARGS) array_to_halfvec(PG_FUNCTION_ARGS)
{ {
@@ -496,7 +519,7 @@ array_to_halfvec(PG_FUNCTION_ARGS)
/* /*
* Convert half vector to float4[] * Convert half vector to float4[]
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_float4); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_float4);
Datum Datum
halfvec_to_float4(PG_FUNCTION_ARGS) halfvec_to_float4(PG_FUNCTION_ARGS)
{ {
@@ -520,7 +543,7 @@ halfvec_to_float4(PG_FUNCTION_ARGS)
/* /*
* Convert vector to half vec * Convert vector to half vec
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_halfvec); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_halfvec);
Datum Datum
vector_to_halfvec(PG_FUNCTION_ARGS) vector_to_halfvec(PG_FUNCTION_ARGS)
{ {
@@ -542,7 +565,7 @@ vector_to_halfvec(PG_FUNCTION_ARGS)
/* /*
* Get the L2 distance between half vectors * Get the L2 distance between half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_distance);
Datum Datum
halfvec_l2_distance(PG_FUNCTION_ARGS) halfvec_l2_distance(PG_FUNCTION_ARGS)
{ {
@@ -557,7 +580,7 @@ halfvec_l2_distance(PG_FUNCTION_ARGS)
/* /*
* Get the L2 squared distance between half vectors * Get the L2 squared distance between half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_squared_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_squared_distance);
Datum Datum
halfvec_l2_squared_distance(PG_FUNCTION_ARGS) halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
{ {
@@ -572,7 +595,7 @@ halfvec_l2_squared_distance(PG_FUNCTION_ARGS)
/* /*
* Get the inner product of two half vectors * Get the inner product of two half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_inner_product);
Datum Datum
halfvec_inner_product(PG_FUNCTION_ARGS) halfvec_inner_product(PG_FUNCTION_ARGS)
{ {
@@ -587,7 +610,7 @@ halfvec_inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the negative inner product of two half vectors * Get the negative inner product of two half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_negative_inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_negative_inner_product);
Datum Datum
halfvec_negative_inner_product(PG_FUNCTION_ARGS) halfvec_negative_inner_product(PG_FUNCTION_ARGS)
{ {
@@ -602,7 +625,7 @@ halfvec_negative_inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the cosine distance between two half vectors * Get the cosine distance between two half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_cosine_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_cosine_distance);
Datum Datum
halfvec_cosine_distance(PG_FUNCTION_ARGS) halfvec_cosine_distance(PG_FUNCTION_ARGS)
{ {
@@ -634,7 +657,7 @@ halfvec_cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality * Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm) * Assumes inputs are unit vectors (skips norm)
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_spherical_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_spherical_distance);
Datum Datum
halfvec_spherical_distance(PG_FUNCTION_ARGS) halfvec_spherical_distance(PG_FUNCTION_ARGS)
{ {
@@ -658,7 +681,7 @@ halfvec_spherical_distance(PG_FUNCTION_ARGS)
/* /*
* Get the L1 distance between two half vectors * Get the L1 distance between two half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l1_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l1_distance);
Datum Datum
halfvec_l1_distance(PG_FUNCTION_ARGS) halfvec_l1_distance(PG_FUNCTION_ARGS)
{ {
@@ -673,7 +696,7 @@ halfvec_l1_distance(PG_FUNCTION_ARGS)
/* /*
* Get the dimensions of a half vector * Get the dimensions of a half vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_vector_dims); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_vector_dims);
Datum Datum
halfvec_vector_dims(PG_FUNCTION_ARGS) halfvec_vector_dims(PG_FUNCTION_ARGS)
{ {
@@ -685,7 +708,7 @@ halfvec_vector_dims(PG_FUNCTION_ARGS)
/* /*
* Get the L2 norm of a half vector * Get the L2 norm of a half vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_norm); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_norm);
Datum Datum
halfvec_l2_norm(PG_FUNCTION_ARGS) halfvec_l2_norm(PG_FUNCTION_ARGS)
{ {
@@ -707,7 +730,7 @@ halfvec_l2_norm(PG_FUNCTION_ARGS)
/* /*
* Normalize a half vector with the L2 norm * Normalize a half vector with the L2 norm
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_l2_normalize); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_l2_normalize);
Datum Datum
halfvec_l2_normalize(PG_FUNCTION_ARGS) halfvec_l2_normalize(PG_FUNCTION_ARGS)
{ {
@@ -746,7 +769,7 @@ halfvec_l2_normalize(PG_FUNCTION_ARGS)
/* /*
* Add half vectors * Add half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_add); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_add);
Datum Datum
halfvec_add(PG_FUNCTION_ARGS) halfvec_add(PG_FUNCTION_ARGS)
{ {
@@ -785,7 +808,7 @@ halfvec_add(PG_FUNCTION_ARGS)
/* /*
* Subtract half vectors * Subtract half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_sub); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_sub);
Datum Datum
halfvec_sub(PG_FUNCTION_ARGS) halfvec_sub(PG_FUNCTION_ARGS)
{ {
@@ -824,7 +847,7 @@ halfvec_sub(PG_FUNCTION_ARGS)
/* /*
* Multiply half vectors * Multiply half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_mul); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_mul);
Datum Datum
halfvec_mul(PG_FUNCTION_ARGS) halfvec_mul(PG_FUNCTION_ARGS)
{ {
@@ -866,7 +889,7 @@ halfvec_mul(PG_FUNCTION_ARGS)
/* /*
* Concatenate half vectors * Concatenate half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_concat); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_concat);
Datum Datum
halfvec_concat(PG_FUNCTION_ARGS) halfvec_concat(PG_FUNCTION_ARGS)
{ {
@@ -890,7 +913,7 @@ halfvec_concat(PG_FUNCTION_ARGS)
/* /*
* Quantize a half vector * Quantize a half vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_binary_quantize); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_binary_quantize);
Datum Datum
halfvec_binary_quantize(PG_FUNCTION_ARGS) halfvec_binary_quantize(PG_FUNCTION_ARGS)
{ {
@@ -908,7 +931,7 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
/* /*
* Get a subvector * Get a subvector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_subvector); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_subvector);
Datum Datum
halfvec_subvector(PG_FUNCTION_ARGS) halfvec_subvector(PG_FUNCTION_ARGS)
{ {
@@ -982,7 +1005,7 @@ halfvec_cmp_internal(HalfVector * a, HalfVector * b)
/* /*
* Less than * Less than
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_lt); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_lt);
Datum Datum
halfvec_lt(PG_FUNCTION_ARGS) halfvec_lt(PG_FUNCTION_ARGS)
{ {
@@ -995,7 +1018,7 @@ halfvec_lt(PG_FUNCTION_ARGS)
/* /*
* Less than or equal * Less than or equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_le); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_le);
Datum Datum
halfvec_le(PG_FUNCTION_ARGS) halfvec_le(PG_FUNCTION_ARGS)
{ {
@@ -1008,7 +1031,7 @@ halfvec_le(PG_FUNCTION_ARGS)
/* /*
* Equal * Equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_eq); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_eq);
Datum Datum
halfvec_eq(PG_FUNCTION_ARGS) halfvec_eq(PG_FUNCTION_ARGS)
{ {
@@ -1021,7 +1044,7 @@ halfvec_eq(PG_FUNCTION_ARGS)
/* /*
* Not equal * Not equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_ne); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ne);
Datum Datum
halfvec_ne(PG_FUNCTION_ARGS) halfvec_ne(PG_FUNCTION_ARGS)
{ {
@@ -1034,7 +1057,7 @@ halfvec_ne(PG_FUNCTION_ARGS)
/* /*
* Greater than or equal * Greater than or equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_ge); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_ge);
Datum Datum
halfvec_ge(PG_FUNCTION_ARGS) halfvec_ge(PG_FUNCTION_ARGS)
{ {
@@ -1047,7 +1070,7 @@ halfvec_ge(PG_FUNCTION_ARGS)
/* /*
* Greater than * Greater than
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_gt); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_gt);
Datum Datum
halfvec_gt(PG_FUNCTION_ARGS) halfvec_gt(PG_FUNCTION_ARGS)
{ {
@@ -1060,7 +1083,7 @@ halfvec_gt(PG_FUNCTION_ARGS)
/* /*
* Compare half vectors * Compare half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_cmp); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_cmp);
Datum Datum
halfvec_cmp(PG_FUNCTION_ARGS) halfvec_cmp(PG_FUNCTION_ARGS)
{ {
@@ -1073,7 +1096,7 @@ halfvec_cmp(PG_FUNCTION_ARGS)
/* /*
* Accumulate half vectors * Accumulate half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_accum); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_accum);
Datum Datum
halfvec_accum(PG_FUNCTION_ARGS) halfvec_accum(PG_FUNCTION_ARGS)
{ {
@@ -1134,7 +1157,7 @@ halfvec_accum(PG_FUNCTION_ARGS)
/* /*
* Average half vectors * Average half vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_avg); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_avg);
Datum Datum
halfvec_avg(PG_FUNCTION_ARGS) halfvec_avg(PG_FUNCTION_ARGS)
{ {
@@ -1168,7 +1191,7 @@ halfvec_avg(PG_FUNCTION_ARGS)
/* /*
* Convert sparse vector to half vector * Convert sparse vector to half vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_halfvec); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_halfvec);
Datum Datum
sparsevec_to_halfvec(PG_FUNCTION_ARGS) sparsevec_to_halfvec(PG_FUNCTION_ARGS)
{ {

View File

@@ -9,7 +9,7 @@
/* TODO Move to better place */ /* TODO Move to better place */
#ifndef DISABLE_DISPATCH #ifndef DISABLE_DISPATCH
/* Only enable for more recent compilers to keep build process simple */ /* Only enable for more recent compilers to keep build process simple */
#if defined(__x86_64__) && defined(__GNUC__) && __GNUC__ >= 9 #if defined(__x86_64__) && defined(__GNUC__) && __GNUC__ >= 8
#define USE_DISPATCH #define USE_DISPATCH
#elif defined(__x86_64__) && defined(__clang_major__) && __clang_major__ >= 7 #elif defined(__x86_64__) && defined(__clang_major__) && __clang_major__ >= 7
#define USE_DISPATCH #define USE_DISPATCH
@@ -19,17 +19,9 @@
#endif #endif
/* target_clones requires glibc */ /* target_clones requires glibc */
#if defined(USE_DISPATCH) && defined(__gnu_linux__) && defined(__has_attribute) #if defined(USE_DISPATCH) && defined(__gnu_linux__)
/* Use separate line for portability */
#if __has_attribute(target_clones)
#define USE_TARGET_CLONES #define USE_TARGET_CLONES
#endif #endif
#endif
/* Apple clang check needed for universal binaries on Mac */
#if defined(USE_DISPATCH) && (defined(HAVE__GET_CPUID) || defined(__apple_build_version__))
#define USE__GET_CPUID
#endif
#if defined(USE_DISPATCH) #if defined(USE_DISPATCH)
#define HALFVEC_DISPATCH #define HALFVEC_DISPATCH
@@ -38,7 +30,7 @@
/* F16C has better performance than _Float16 (on x86-64) */ /* F16C has better performance than _Float16 (on x86-64) */
#if defined(__F16C__) #if defined(__F16C__)
#define F16C_SUPPORT #define F16C_SUPPORT
#elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH) && !defined(__FreeBSD__) && (!defined(__i386__) || defined(__SSE2__)) #elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH)
#define FLT16_SUPPORT #define FLT16_SUPPORT
#endif #endif

View File

@@ -9,10 +9,8 @@
#include "commands/vacuum.h" #include "commands/vacuum.h"
#include "hnsw.h" #include "hnsw.h"
#include "miscadmin.h" #include "miscadmin.h"
#include "utils/float.h"
#include "utils/guc.h" #include "utils/guc.h"
#include "utils/selfuncs.h" #include "utils/selfuncs.h"
#include "utils/spccache.h"
#if PG_VERSION_NUM < 150000 #if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x) #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -61,9 +59,17 @@ HnswInit(void)
hnsw_relopt_kind = add_reloption_kind(); hnsw_relopt_kind = add_reloption_kind();
add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections", add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",
HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M, AccessExclusiveLock); HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
add_int_reloption(hnsw_relopt_kind, "ef_construction", "Size of the dynamic candidate list for construction", add_int_reloption(hnsw_relopt_kind, "ef_construction", "Size of the dynamic candidate list for construction",
HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION, AccessExclusiveLock); HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search", DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
"Valid range is 1..1000.", &hnsw_ef_search, "Valid range is 1..1000.", &hnsw_ef_search,
@@ -100,16 +106,14 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
{ {
GenericCosts costs; GenericCosts costs;
int m; int m;
double ratio; int entryLevel;
double startupPages;
double spc_seq_page_cost;
Relation index; Relation index;
/* Never use index without order */ /* Never use index without order */
if (path->indexorderbys == NULL) if (path->indexorderbys == NULL)
{ {
*indexStartupCost = get_float8_infinity(); *indexStartupCost = DBL_MAX;
*indexTotalCost = get_float8_infinity(); *indexTotalCost = DBL_MAX;
*indexSelectivity = 0; *indexSelectivity = 0;
*indexCorrelation = 0; *indexCorrelation = 0;
*indexPages = 0; *indexPages = 0;
@@ -118,71 +122,21 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
MemSet(&costs, 0, sizeof(costs)); MemSet(&costs, 0, sizeof(costs));
genericcostestimate(root, path, loop_count, &costs);
index = index_open(path->indexinfo->indexoid, NoLock); index = index_open(path->indexinfo->indexoid, NoLock);
HnswGetMetaPageInfo(index, &m, NULL); HnswGetMetaPageInfo(index, &m, NULL);
index_close(index, NoLock); index_close(index, NoLock);
/* /* Approximate entry level */
* HNSW cost estimation follows a formula that accounts for the total entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
* number of tuples indexed combined with the parameters that most
* influence the duration of the index scan, namely: m - the number of
* tuples that are scanned in each step of the HNSW graph traversal
* ef_search - which influences the total number of steps taken at layer 0
*
* The source of the vector data can impact how many steps it takes to
* converge on the set of vectors to return to the executor. Currently, we
* use a hardcoded scaling factor (HNSWScanScalingFactor) to help
* influence that, but this could later become a configurable parameter
* based on the cost estimations.
*
* The tuple estimator formula is below:
*
* numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
*
* "entryLevel * m" represents the floor of tuples we need to scan to get
* to layer 0 (L0).
*
* "layer0TuplesMax" is the estimated total number of tuples we'd scan at
* L0 if we weren't discarding already visited tuples as part of the scan.
*
* "layer0Selectivity" estimates the percentage of tuples that are scanned
* at L0, accounting for previously visited tuples, multiplied by the
* "scalingFactor" (currently hardcoded).
*/
if (path->indexinfo->tuples > 0)
{
double scalingFactor = 0.55;
int entryLevel = (int) (log(path->indexinfo->tuples) * HnswGetMl(m));
int layer0TuplesMax = HnswGetLayerM(m, 0) * hnsw_ef_search;
double layer0Selectivity = scalingFactor * log(path->indexinfo->tuples) / (log(m) * (1 + log(hnsw_ef_search)));
ratio = (entryLevel * m + layer0TuplesMax * layer0Selectivity) / path->indexinfo->tuples; /* TODO Improve estimate of visited tuples (currently underestimates) */
/* Account for number of tuples (or entry level), m, and ef_search */
costs.numIndexTuples = (entryLevel + 2) * m;
if (ratio > 1) genericcostestimate(root, path, loop_count, &costs);
ratio = 1;
}
else
ratio = 1;
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost); /* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost;
/* Startup cost is cost before returning the first row */
costs.indexStartupCost = costs.indexTotalCost * ratio;
/* Adjust cost if needed since TOAST not included in seq scan cost */
startupPages = costs.numIndexPages * ratio;
if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
{
/* Change all page cost from random to sequential */
costs.indexStartupCost -= startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Remove cost of extra pages */
costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
}
*indexStartupCost = costs.indexStartupCost;
*indexTotalCost = costs.indexTotalCost; *indexTotalCost = costs.indexTotalCost;
*indexSelectivity = costs.indexSelectivity; *indexSelectivity = costs.indexSelectivity;
*indexCorrelation = costs.indexCorrelation; *indexCorrelation = costs.indexCorrelation;
@@ -200,10 +154,23 @@ hnswoptions(Datum reloptions, bool validate)
{"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)}, {"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)},
}; };
#if PG_VERSION_NUM >= 130000
return (bytea *) build_reloptions(reloptions, validate, return (bytea *) build_reloptions(reloptions, validate,
hnsw_relopt_kind, hnsw_relopt_kind,
sizeof(HnswOptions), sizeof(HnswOptions),
tab, lengthof(tab)); tab, lengthof(tab));
#else
relopt_value *options;
int numoptions;
HnswOptions *rdopts;
options = parseRelOptions(reloptions, validate, hnsw_relopt_kind, &numoptions);
rdopts = allocateReloptStruct(sizeof(HnswOptions), options, numoptions);
fillRelOptions((void *) rdopts, sizeof(HnswOptions), options, numoptions,
validate, tab, lengthof(tab));
return (bytea *) rdopts;
#endif
} }
/* /*
@@ -220,20 +187,22 @@ hnswvalidate(Oid opclassoid)
* *
* See https://www.postgresql.org/docs/current/index-api.html * See https://www.postgresql.org/docs/current/index-api.html
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler); PGDLLEXPORT PG_FUNCTION_INFO_V1(hnswhandler);
Datum Datum
hnswhandler(PG_FUNCTION_ARGS) hnswhandler(PG_FUNCTION_ARGS)
{ {
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine); IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0; amroutine->amstrategies = 0;
amroutine->amsupport = 4; amroutine->amsupport = 3;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0; amroutine->amoptsprocnum = 0;
#endif
amroutine->amcanorder = false; amroutine->amcanorder = false;
amroutine->amcanorderbyop = true; amroutine->amcanorderbyop = true;
amroutine->amcanbackward = false; /* can change direction mid-scan */ amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false; amroutine->amcanunique = false;
amroutine->amcanmulticol = true; amroutine->amcanmulticol = false;
amroutine->amoptionalkey = true; amroutine->amoptionalkey = true;
amroutine->amsearcharray = false; amroutine->amsearcharray = false;
amroutine->amsearchnulls = false; amroutine->amsearchnulls = false;
@@ -241,24 +210,17 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amclusterable = false; amroutine->amclusterable = false;
amroutine->ampredlocks = false; amroutine->ampredlocks = false;
amroutine->amcanparallel = false; amroutine->amcanparallel = false;
#if PG_VERSION_NUM >= 170000
amroutine->amcanbuildparallel = true;
#endif
amroutine->amcaninclude = false; amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */ amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
#if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false;
#endif
amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL; amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
#endif
amroutine->amkeytype = InvalidOid; amroutine->amkeytype = InvalidOid;
/* Interface functions */ /* Interface functions */
amroutine->ambuild = hnswbuild; amroutine->ambuild = hnswbuild;
amroutine->ambuildempty = hnswbuildempty; amroutine->ambuildempty = hnswbuildempty;
amroutine->aminsert = hnswinsert; amroutine->aminsert = hnswinsert;
#if PG_VERSION_NUM >= 170000
amroutine->aminsertcleanup = NULL;
#endif
amroutine->ambulkdelete = hnswbulkdelete; amroutine->ambulkdelete = hnswbulkdelete;
amroutine->amvacuumcleanup = hnswvacuumcleanup; amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL; amroutine->amcanreturn = NULL;
@@ -285,17 +247,3 @@ hnswhandler(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(amroutine); PG_RETURN_POINTER(amroutine);
} }
/*
* Get the distance between two int4 attributes
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_int4_attribute_distance);
Datum
hnsw_int4_attribute_distance(PG_FUNCTION_ARGS)
{
int32 a = PG_GETARG_INT32(0);
int32 b = PG_GETARG_INT32(1);
double distance = ((double) a) - ((double) b);
PG_RETURN_FLOAT8(distance);
}

View File

@@ -19,7 +19,6 @@
#define HNSW_DISTANCE_PROC 1 #define HNSW_DISTANCE_PROC 1
#define HNSW_NORM_PROC 2 #define HNSW_NORM_PROC 2
#define HNSW_TYPE_INFO_PROC 3 #define HNSW_TYPE_INFO_PROC 3
#define HNSW_ATTRIBUTE_DISTANCE_PROC 4
#define HNSW_VERSION 1 #define HNSW_VERSION 1
#define HNSW_MAGIC_NUMBER 0xA953A953 #define HNSW_MAGIC_NUMBER 0xA953A953
@@ -77,6 +76,11 @@
#define SeedRandom(seed) srandom(seed) #define SeedRandom(seed) srandom(seed)
#endif #endif
#if PG_VERSION_NUM < 130000
#define list_delete_last(list) list_truncate(list, list_length(list) - 1)
#define list_sort(list, cmp) ((list) = list_qsort(list, cmp))
#endif
#define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE) #define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
#define HnswIsNeighborTuple(tup) ((tup)->type == HNSW_NEIGHBOR_TUPLE_TYPE) #define HnswIsNeighborTuple(tup) ((tup)->type == HNSW_NEIGHBOR_TUPLE_TYPE)
@@ -105,8 +109,6 @@
#define HnswPtrPointer(hp) (hp).ptr #define HnswPtrPointer(hp) (hp).ptr
#define HnswPtrOffset(hp) relptr_offset((hp).relptr) #define HnswPtrOffset(hp) relptr_offset((hp).relptr)
#define HnswUseIndexTuple(index) (IndexRelationGetNumberOfAttributes(index) > 1)
/* Variables */ /* Variables */
extern int hnsw_ef_search; extern int hnsw_ef_search;
extern int hnsw_lock_tranche_id; extern int hnsw_lock_tranche_id;
@@ -124,7 +126,6 @@ HnswPtrDeclare(HnswElementData, HnswElementRelptr, HnswElementPtr);
HnswPtrDeclare(HnswNeighborArray, HnswNeighborArrayRelptr, HnswNeighborArrayPtr); HnswPtrDeclare(HnswNeighborArray, HnswNeighborArrayRelptr, HnswNeighborArrayPtr);
HnswPtrDeclare(HnswNeighborArrayPtr, HnswNeighborsRelptr, HnswNeighborsPtr); HnswPtrDeclare(HnswNeighborArrayPtr, HnswNeighborsRelptr, HnswNeighborsPtr);
HnswPtrDeclare(char, DatumRelptr, DatumPtr); HnswPtrDeclare(char, DatumRelptr, DatumPtr);
HnswPtrDeclare(IndexTupleData, IndexTupleRelptr, IndexTuplePtr);
struct HnswElementData struct HnswElementData
{ {
@@ -140,7 +141,6 @@ struct HnswElementData
OffsetNumber neighborOffno; OffsetNumber neighborOffno;
BlockNumber neighborPage; BlockNumber neighborPage;
DatumPtr value; DatumPtr value;
IndexTuplePtr itup;
LWLock lock; LWLock lock;
}; };
@@ -160,14 +160,11 @@ struct HnswNeighborArray
HnswCandidate items[FLEXIBLE_ARRAY_MEMBER]; HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
}; };
typedef struct HnswSearchCandidate typedef struct HnswPairingHeapNode
{ {
pairingheap_node c_node; pairingheap_node ph_node;
pairingheap_node w_node; HnswCandidate *inner;
HnswElementPtr element; } HnswPairingHeapNode;
double distance;
bool matches;
} HnswSearchCandidate;
/* HNSW index options */ /* HNSW index options */
typedef struct HnswOptions typedef struct HnswOptions
@@ -191,8 +188,8 @@ typedef struct HnswGraph
/* Allocations state */ /* Allocations state */
LWLock allocatorLock; LWLock allocatorLock;
Size memoryUsed; long memoryUsed;
Size memoryTotal; long memoryTotal;
/* Flushed state */ /* Flushed state */
LWLock flushLock; LWLock flushLock;
@@ -262,17 +259,15 @@ typedef struct HnswBuildState
double reltuples; double reltuples;
/* Support functions */ /* Support functions */
FmgrInfo *procinfo[2]; FmgrInfo *procinfo;
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
Oid *collation; Oid collation;
/* Variables */ /* Variables */
HnswGraph graphData; HnswGraph graphData;
HnswGraph *graph; HnswGraph *graph;
double ml; double ml;
int maxLevel; int maxLevel;
bool useIndexTuple;
TupleDesc tupdesc;
/* Memory */ /* Memory */
MemoryContext graphCtx; MemoryContext graphCtx;
@@ -341,9 +336,9 @@ typedef struct HnswScanOpaqueData
MemoryContext tmpCtx; MemoryContext tmpCtx;
/* Support functions */ /* Support functions */
FmgrInfo *procinfo[2]; FmgrInfo *procinfo;
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
Oid *collation; Oid collation;
} HnswScanOpaqueData; } HnswScanOpaqueData;
typedef HnswScanOpaqueData * HnswScanOpaque; typedef HnswScanOpaqueData * HnswScanOpaque;
@@ -361,8 +356,8 @@ typedef struct HnswVacuumState
int efConstruction; int efConstruction;
/* Support functions */ /* Support functions */
FmgrInfo *procinfo[2]; FmgrInfo *procinfo;
Oid *collation; Oid collation;
/* Variables */ /* Variables */
struct tidhash_hash *deleted; struct tidhash_hash *deleted;
@@ -383,32 +378,28 @@ bool HnswCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum); Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page); void HnswInitPage(Buffer buf, Page page);
void HnswInit(void); void HnswInit(void);
List *HnswSearchLayer(char *base, Datum q, IndexTuple qtup, ScanKeyData *keyData, List *ep, int ef, int lc, Relation index, FmgrInfo **procinfo, Oid *collation, int m, bool inserting, HnswElement skipElement, bool inMemory); List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
HnswElement HnswGetEntryPoint(Relation index); HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint); void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
void *HnswAlloc(HnswAllocator * allocator, Size size); void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc); HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno); HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo **procinfo, Oid *collation, int m, int efConstruction, bool existing, bool inMemory); void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, IndexTuple qtup, ScanKeyData *keyData, Relation rel, FmgrInfo **procinfo, Oid *collation, bool loadVec, bool inMemory); HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building); void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m); void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid); void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
HnswNeighborArray *HnswInitNeighborArray(int lm, HnswAllocator * allocator);
void HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * alloc); void HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * alloc);
bool HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building); bool HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
void HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement e, int m, bool checkExisting, bool building); void HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec, Relation index); void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
void HnswLoadElement(HnswElement element, double *distance, bool *matches, Datum *q, IndexTuple qtup, ScanKeyData *keyData, Relation index, FmgrInfo **procinfo, Oid *collation, bool loadVec, double *maxDistance); void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element, bool useIndexTuple); void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
void HnswUpdateConnection(char *base, HnswNeighborArray * neighbors, HnswElement newElement, float distance, int lm, int *updateIdx, Relation index, FmgrInfo **procinfo, Oid *collation); void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
bool HnswLoadNeighborTids(HnswElement element, ItemPointerData *indextids, Relation index, int m, int lm, int lc); void HnswLoadNeighbors(HnswElement element, Relation index, int m);
void HnswInitLockTranche(void); void HnswInitLockTranche(void);
const HnswTypeInfo *HnswGetTypeInfo(Relation index); const HnswTypeInfo *HnswGetTypeInfo(Relation index);
PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc); PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
void HnswInitProcinfo(FmgrInfo **procinfo, Oid **collation, Relation index);
Size HnswGetElementTupleSize(char *base, HnswElement element, bool useIndexTuple);
bool HnswIndexTupleIsEqual(IndexTuple a, IndexTuple b, TupleDesc tupdesc);
/* Index access methods */ /* Index access methods */
IndexBuildResult *hnswbuild(Relation heap, Relation index, IndexInfo *indexInfo); IndexBuildResult *hnswbuild(Relation heap, Relation index, IndexInfo *indexInfo);

View File

@@ -60,6 +60,12 @@
#include "pgstat.h" #include "pgstat.h"
#endif #endif
#if PG_VERSION_NUM >= 130000
#define CALLBACK_ITEM_POINTER ItemPointer tid
#else
#define CALLBACK_ITEM_POINTER HeapTuple hup
#endif
#if PG_VERSION_NUM >= 140000 #if PG_VERSION_NUM >= 140000
#include "utils/backend_status.h" #include "utils/backend_status.h"
#include "utils/wait_event.h" #include "utils/wait_event.h"
@@ -69,6 +75,10 @@
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002) #define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003) #define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
#if PG_VERSION_NUM < 130000
#define GENERATIONCHUNK_RAWSIZE (SIZEOF_SIZE_T + SIZEOF_VOID_P * 2)
#endif
/* /*
* Create the metapage * Create the metapage
*/ */
@@ -148,7 +158,6 @@ CreateGraphPages(HnswBuildState * buildstate)
Page page; Page page;
HnswElementPtr iter = buildstate->graph->head; HnswElementPtr iter = buildstate->graph->head;
char *base = buildstate->hnswarea; char *base = buildstate->hnswarea;
bool useIndexTuple = buildstate->useIndexTuple;
/* Calculate sizes */ /* Calculate sizes */
maxSize = HNSW_MAX_SIZE; maxSize = HNSW_MAX_SIZE;
@@ -168,6 +177,7 @@ CreateGraphPages(HnswBuildState * buildstate)
Size etupSize; Size etupSize;
Size ntupSize; Size ntupSize;
Size combinedSize; Size combinedSize;
Pointer valuePtr = HnswPtrAccess(base, element->value);
/* Update iterator */ /* Update iterator */
iter = element->next; iter = element->next;
@@ -176,17 +186,15 @@ CreateGraphPages(HnswBuildState * buildstate)
MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE); MemSet(etup, 0, HNSW_TUPLE_ALLOC_SIZE);
/* Calculate sizes */ /* Calculate sizes */
etupSize = HnswGetElementTupleSize(base, element, useIndexTuple); etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(valuePtr));
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, buildstate->m); ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, buildstate->m);
combinedSize = etupSize + ntupSize + sizeof(ItemIdData); combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
/* Initial size check */ /* Initial size check */
if (etupSize > HNSW_TUPLE_ALLOC_SIZE) if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
ereport(ERROR, elog(ERROR, "index tuple too large");
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("index tuple too large")));
HnswSetElementTuple(base, etup, element, useIndexTuple); HnswSetElementTuple(base, etup, element);
/* Keep element and neighbors on the same page if possible */ /* Keep element and neighbors on the same page if possible */
if (PageGetFreeSpace(page) < etupSize || (combinedSize <= maxSize && PageGetFreeSpace(page) < combinedSize)) if (PageGetFreeSpace(page) < etupSize || (combinedSize <= maxSize && PageGetFreeSpace(page) < combinedSize))
@@ -327,29 +335,20 @@ AddDuplicateInMemory(HnswElement element, HnswElement dup)
* Find duplicate element * Find duplicate element
*/ */
static bool static bool
FindDuplicateInMemory(char *base, HnswElement element, bool useIndexTuple, TupleDesc tupdesc) FindDuplicateInMemory(char *base, HnswElement element)
{ {
HnswNeighborArray *neighbors = HnswGetNeighbors(base, element, 0); HnswNeighborArray *neighbors = HnswGetNeighbors(base, element, 0);
Datum value = HnswGetValue(base, element); Datum value = HnswGetValue(base, element);
IndexTuple itup = HnswPtrAccess(base, element->itup);
for (int i = 0; i < neighbors->length; i++) for (int i = 0; i < neighbors->length; i++)
{ {
HnswCandidate *neighbor = &neighbors->items[i]; HnswCandidate *neighbor = &neighbors->items[i];
HnswElement neighborElement = HnswPtrAccess(base, neighbor->element); HnswElement neighborElement = HnswPtrAccess(base, neighbor->element);
Datum neighborValue = HnswGetValue(base, neighborElement);
if (useIndexTuple) /* Exit early since ordered by distance */
{ if (!datumIsEqual(value, neighborValue, false, -1))
/* Exit early since ordered by distance */ return false;
if (!HnswIndexTupleIsEqual(itup, HnswPtrAccess(base, neighborElement->itup), tupdesc))
return false;
}
else
{
/* Exit early since ordered by distance */
if (!datumIsEqual(value, HnswGetValue(base, neighborElement), false, -1))
return false;
}
/* Check for space */ /* Check for space */
if (AddDuplicateInMemory(element, neighborElement)) if (AddDuplicateInMemory(element, neighborElement))
@@ -375,18 +374,12 @@ AddElementInMemory(char *base, HnswGraph * graph, HnswElement element)
* Update neighbors * Update neighbors
*/ */
static void static void
UpdateNeighborsInMemory(char *base, Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement e, int m) UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswElement e, int m)
{ {
for (int lc = e->level; lc >= 0; lc--) for (int lc = e->level; lc >= 0; lc--)
{ {
int lm = HnswGetLayerM(m, lc); int lm = HnswGetLayerM(m, lc);
Size neighborsSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm); HnswNeighborArray *neighbors = HnswGetNeighbors(base, e, lc);
HnswNeighborArray *neighbors = palloc(neighborsSize);
/* Copy neighbors to local memory */
LWLockAcquire(&e->lock, LW_SHARED);
memcpy(neighbors, HnswGetNeighbors(base, e, lc), neighborsSize);
LWLockRelease(&e->lock);
for (int i = 0; i < neighbors->length; i++) for (int i = 0; i < neighbors->length; i++)
{ {
@@ -396,8 +389,9 @@ UpdateNeighborsInMemory(char *base, Relation index, FmgrInfo **procinfo, Oid *co
/* Keep scan-build happy on Mac x86-64 */ /* Keep scan-build happy on Mac x86-64 */
Assert(neighborElement); Assert(neighborElement);
/* Use element for lock instead of hc since hc can be replaced */
LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE); LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
HnswUpdateConnection(base, HnswGetNeighbors(base, neighborElement, lc), e, hc->distance, lm, NULL, index, procinfo, collation); HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
LWLockRelease(&neighborElement->lock); LWLockRelease(&neighborElement->lock);
} }
} }
@@ -407,20 +401,20 @@ UpdateNeighborsInMemory(char *base, Relation index, FmgrInfo **procinfo, Oid *co
* Update graph in memory * Update graph in memory
*/ */
static void static void
UpdateGraphInMemory(FmgrInfo **procinfo, Oid *collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate) UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
{ {
HnswGraph *graph = buildstate->graph; HnswGraph *graph = buildstate->graph;
char *base = buildstate->hnswarea; char *base = buildstate->hnswarea;
/* Look for duplicate */ /* Look for duplicate */
if (FindDuplicateInMemory(base, element, buildstate->useIndexTuple, buildstate->tupdesc)) if (FindDuplicateInMemory(base, element))
return; return;
/* Add element */ /* Add element */
AddElementInMemory(base, graph, element); AddElementInMemory(base, graph, element);
/* Update neighbors */ /* Update neighbors */
UpdateNeighborsInMemory(base, buildstate->index, procinfo, collation, element, m); UpdateNeighborsInMemory(base, procinfo, collation, element, m);
/* Update entry point if needed (already have lock) */ /* Update entry point if needed (already have lock) */
if (entryPoint == NULL || element->level > entryPoint->level) if (entryPoint == NULL || element->level > entryPoint->level)
@@ -433,9 +427,8 @@ UpdateGraphInMemory(FmgrInfo **procinfo, Oid *collation, HnswElement element, in
static void static void
InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element) InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
{ {
Relation index = buildstate->index; FmgrInfo *procinfo = buildstate->procinfo;
FmgrInfo **procinfo = buildstate->procinfo; Oid collation = buildstate->collation;
Oid *collation = buildstate->collation;
HnswGraph *graph = buildstate->graph; HnswGraph *graph = buildstate->graph;
HnswElement entryPoint; HnswElement entryPoint;
LWLock *entryLock = &graph->entryLock; LWLock *entryLock = &graph->entryLock;
@@ -468,7 +461,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
} }
/* Find neighbors for element */ /* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false, true); HnswFindElementNeighbors(base, element, entryPoint, NULL, procinfo, collation, m, efConstruction, false);
/* Update graph in memory */ /* Update graph in memory */
UpdateGraphInMemory(procinfo, collation, element, m, efConstruction, entryPoint, buildstate); UpdateGraphInMemory(procinfo, collation, element, m, efConstruction, entryPoint, buildstate);
@@ -491,11 +484,6 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
Pointer valuePtr; Pointer valuePtr;
LWLock *flushLock = &graph->flushLock; LWLock *flushLock = &graph->flushLock;
char *base = buildstate->hnswarea; char *base = buildstate->hnswarea;
bool useIndexTuple = buildstate->useIndexTuple;
TupleDesc tupdesc = buildstate->tupdesc;
IndexTuple itup;
Size itupSize;
IndexTuple itupPtr;
/* Detoast once for all calls */ /* Detoast once for all calls */
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
@@ -507,10 +495,10 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Normalize if needed */ /* Normalize if needed */
if (buildstate->normprocinfo != NULL) if (buildstate->normprocinfo != NULL)
{ {
if (!HnswCheckNorm(buildstate->normprocinfo, buildstate->collation[0], value)) if (!HnswCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
return false; return false;
value = HnswNormValue(typeInfo, buildstate->collation[0], value); value = HnswNormValue(typeInfo, buildstate->collation, value);
} }
/* Get datum size */ /* Get datum size */
@@ -561,17 +549,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Ok, we can proceed to allocate the element */ /* Ok, we can proceed to allocate the element */
element = HnswInitElement(base, heaptid, buildstate->m, buildstate->ml, buildstate->maxLevel, allocator); element = HnswInitElement(base, heaptid, buildstate->m, buildstate->ml, buildstate->maxLevel, allocator);
valuePtr = HnswAlloc(allocator, valueSize);
if (useIndexTuple)
{
/* TODO fix */
values[0] = value;
itup = index_form_tuple(tupdesc, values, isnull);
itupSize = IndexTupleSize(itup);
itupPtr = HnswAlloc(allocator, itupSize);
}
else
valuePtr = HnswAlloc(allocator, valueSize);
/* /*
* We have now allocated the space needed for the element, so we don't * We have now allocated the space needed for the element, so we don't
@@ -581,19 +559,8 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
LWLockRelease(&graph->allocatorLock); LWLockRelease(&graph->allocatorLock);
/* Copy the datum */ /* Copy the datum */
if (useIndexTuple) memcpy(valuePtr, DatumGetPointer(value), valueSize);
{ HnswPtrStore(base, element->value, valuePtr);
bool unused;
memcpy(itupPtr, itup, itupSize);
HnswPtrStore(base, element->itup, itupPtr);
HnswPtrStore(base, element->value, DatumGetPointer(index_getattr(itupPtr, 1, tupdesc, &unused)));
}
else
{
memcpy(valuePtr, DatumGetPointer(value), valueSize);
HnswPtrStore(base, element->value, valuePtr);
}
/* Create a lock for the element */ /* Create a lock for the element */
LWLockInitialize(&element->lock, hnsw_lock_tranche_id); LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -611,13 +578,17 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Callback for table_index_build_scan * Callback for table_index_build_scan
*/ */
static void static void
BuildCallback(Relation index, ItemPointer tid, Datum *values, BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state) bool *isnull, bool tupleIsAlive, void *state)
{ {
HnswBuildState *buildstate = (HnswBuildState *) state; HnswBuildState *buildstate = (HnswBuildState *) state;
HnswGraph *graph = buildstate->graph; HnswGraph *graph = buildstate->graph;
MemoryContext oldCtx; MemoryContext oldCtx;
#if PG_VERSION_NUM < 130000
ItemPointer tid = &hup->t_self;
#endif
/* Skip nulls */ /* Skip nulls */
if (isnull[0]) if (isnull[0])
return; return;
@@ -643,7 +614,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
* Initialize the graph * Initialize the graph
*/ */
static void static void
InitGraph(HnswGraph * graph, char *base, Size memoryTotal) InitGraph(HnswGraph * graph, char *base, long memoryTotal)
{ {
/* Initialize the lock tranche if needed */ /* Initialize the lock tranche if needed */
HnswInitLockTranche(); HnswInitLockTranche();
@@ -680,7 +651,11 @@ HnswMemoryContextAlloc(Size size, void *state)
HnswBuildState *buildstate = (HnswBuildState *) state; HnswBuildState *buildstate = (HnswBuildState *) state;
void *chunk = MemoryContextAlloc(buildstate->graphCtx, size); void *chunk = MemoryContextAlloc(buildstate->graphCtx, size);
#if PG_VERSION_NUM >= 130000
buildstate->graphData.memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false); buildstate->graphData.memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->graphData.memoryUsed += MAXALIGN(size);
#endif
return chunk; return chunk;
} }
@@ -716,52 +691,30 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Disallow varbit since require fixed dimensions */ /* Disallow varbit since require fixed dimensions */
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID) if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
ereport(ERROR, elog(ERROR, "type not supported for hnsw index");
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("type not supported for hnsw index")));
/* TODO See if needed */
if (IndexRelationGetNumberOfKeyAttributes(index) > 2)
elog(ERROR, "index cannot have more than two columns");
if (!OidIsValid(index_getprocid(index, 1, HNSW_DISTANCE_PROC)))
elog(ERROR, "first column must be a vector");
for (int i = 1; i < IndexRelationGetNumberOfKeyAttributes(index); i++)
{
if (!OidIsValid(index_getprocid(index, i + 1, HNSW_ATTRIBUTE_DISTANCE_PROC)))
elog(ERROR, "column %d cannot be a vector", i + 1);
}
/* Require column to have dimensions to be indexed */ /* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0) if (buildstate->dimensions < 0)
ereport(ERROR, elog(ERROR, "column does not have dimensions");
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("column does not have dimensions")));
if (buildstate->dimensions > buildstate->typeInfo->maxDimensions) if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
ereport(ERROR, elog(ERROR, "column cannot have more than %d dimensions for hnsw index", buildstate->typeInfo->maxDimensions);
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("column cannot have more than %d dimensions for hnsw index", buildstate->typeInfo->maxDimensions)));
if (buildstate->efConstruction < 2 * buildstate->m) if (buildstate->efConstruction < 2 * buildstate->m)
ereport(ERROR, elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("ef_construction must be greater than or equal to 2 * m")));
buildstate->reltuples = 0; buildstate->reltuples = 0;
buildstate->indtuples = 0; buildstate->indtuples = 0;
/* Get support functions */ /* Get support functions */
HnswInitProcinfo(buildstate->procinfo, &buildstate->collation, index); buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC); buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
buildstate->collation = index->rd_indcollation[0];
InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L); InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * 1024L);
buildstate->graph = &buildstate->graphData; buildstate->graph = &buildstate->graphData;
buildstate->ml = HnswGetMl(buildstate->m); buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m); buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
buildstate->useIndexTuple = HnswUseIndexTuple(index);
buildstate->tupdesc = RelationGetDescr(index);
buildstate->graphCtx = GenerationContextCreate(CurrentMemoryContext, buildstate->graphCtx = GenerationContextCreate(CurrentMemoryContext,
"Hnsw build graph context", "Hnsw build graph context",
@@ -1168,8 +1121,8 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
BuildGraph(buildstate, forkNum); BuildGraph(buildstate, forkNum);
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM) if (RelationNeedsWAL(index))
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true); log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocks(index), true);
FreeBuildState(buildstate); FreeBuildState(buildstate);
} }

View File

@@ -36,15 +36,14 @@ GetInsertPage(Relation index)
* Check for a free offset * Check for a free offset
*/ */
static bool static bool
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage) HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
{ {
OffsetNumber offno; OffsetNumber offno;
OffsetNumber maxoffno = PageGetMaxOffsetNumber(page); OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno)) for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{ {
ItemId eitemid = PageGetItemId(page, offno); HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, eitemid);
/* Skip neighbor tuples */ /* Skip neighbor tuples */
if (!HnswIsElementTuple(etup)) if (!HnswIsElementTuple(etup))
@@ -55,9 +54,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
BlockNumber elementPage = BufferGetBlockNumber(buf); BlockNumber elementPage = BufferGetBlockNumber(buf);
BlockNumber neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid); BlockNumber neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
OffsetNumber neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid); OffsetNumber neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
ItemId nitemid; ItemId itemid;
Size pageFree;
Size npageFree;
if (!BlockNumberIsValid(*newInsertPage)) if (!BlockNumberIsValid(*newInsertPage))
*newInsertPage = elementPage; *newInsertPage = elementPage;
@@ -76,25 +73,10 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
*npage = BufferGetPage(*nbuf); *npage = BufferGetPage(*nbuf);
} }
nitemid = PageGetItemId(*npage, neighborOffno); itemid = PageGetItemId(*npage, neighborOffno);
/* Ensure aligned for space check */ /* Check for space on neighbor tuple page */
Assert(etupSize == MAXALIGN(etupSize)); if (PageGetFreeSpace(*npage) + ItemIdGetLength(itemid) - sizeof(ItemIdData) >= ntupSize)
Assert(ntupSize == MAXALIGN(ntupSize));
/*
* Calculate free space individually since tuples are overwritten
* individually (in separate calls to PageIndexTupleOverwrite)
*/
pageFree = ItemIdGetLength(eitemid) + PageGetExactFreeSpace(page);
npageFree = ItemIdGetLength(nitemid);
if (neighborPage != elementPage)
npageFree += PageGetExactFreeSpace(*npage);
else if (pageFree >= etupSize)
npageFree += pageFree - etupSize;
/* Check for space */
if (pageFree >= etupSize && npageFree >= ntupSize)
{ {
*freeOffno = offno; *freeOffno = offno;
*freeNeighborOffno = neighborOffno; *freeNeighborOffno = neighborOffno;
@@ -154,10 +136,9 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
OffsetNumber freeNeighborOffno = InvalidOffsetNumber; OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
BlockNumber newInsertPage = InvalidBlockNumber; BlockNumber newInsertPage = InvalidBlockNumber;
char *base = NULL; char *base = NULL;
bool useIndexTuple = HnswUseIndexTuple(index);
/* Calculate sizes */ /* Calculate sizes */
etupSize = HnswGetElementTupleSize(base, e, useIndexTuple); etupSize = HNSW_ELEMENT_TUPLE_SIZE(VARSIZE_ANY(HnswPtrAccess(base, e->value)));
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m); ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m);
combinedSize = etupSize + ntupSize + sizeof(ItemIdData); combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
maxSize = HNSW_MAX_SIZE; maxSize = HNSW_MAX_SIZE;
@@ -165,7 +146,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
/* Prepare element tuple */ /* Prepare element tuple */
etup = palloc0(etupSize); etup = palloc0(etupSize);
HnswSetElementTuple(base, etup, e, useIndexTuple); HnswSetElementTuple(base, etup, e);
/* Prepare neighbor tuple */ /* Prepare neighbor tuple */
ntup = palloc0(ntupSize); ntup = palloc0(ntupSize);
@@ -203,7 +184,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
} }
/* Next, try space from a deleted element */ /* Next, try space from a deleted element */
if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage)) if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
{ {
if (nbuf != buf) if (nbuf != buf)
{ {
@@ -335,107 +316,6 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
*updatedInsertPage = newInsertPage; *updatedInsertPage = newInsertPage;
} }
/*
* Load neighbors
*/
static HnswNeighborArray *
HnswLoadNeighbors(HnswElement element, Relation index, int m, int lm, int lc)
{
char *base = NULL;
HnswNeighborArray *neighbors = HnswInitNeighborArray(lm, NULL);
ItemPointerData indextids[HNSW_MAX_M * 2];
if (!HnswLoadNeighborTids(element, indextids, index, m, lm, lc))
return neighbors;
for (int i = 0; i < lm; i++)
{
ItemPointer indextid = &indextids[i];
HnswElement e;
HnswCandidate *hc;
if (!ItemPointerIsValid(indextid))
break;
e = HnswInitElementFromBlock(ItemPointerGetBlockNumber(indextid), ItemPointerGetOffsetNumber(indextid));
hc = &neighbors->items[neighbors->length++];
HnswPtrStore(base, hc->element, e);
}
return neighbors;
}
/*
* Load elements for insert
*/
static void
LoadElementsForInsert(HnswNeighborArray * neighbors, Datum q, IndexTuple qtup, int *idx, Relation index, FmgrInfo **procinfo, Oid *collation)
{
char *base = NULL;
for (int i = 0; i < neighbors->length; i++)
{
HnswCandidate *hc = &neighbors->items[i];
HnswElement element = HnswPtrAccess(base, hc->element);
double distance;
bool matches;
HnswLoadElement(element, &distance, &matches, &q, qtup, NULL, index, procinfo, collation, true, NULL);
hc->distance = distance;
/* Prune element if being deleted */
if (element->heaptidsLength == 0)
{
*idx = i;
break;
}
}
}
/*
* Get update index
*/
static int
GetUpdateIndex(HnswElement element, HnswElement newElement, float distance, int m, int lm, int lc, Relation index, FmgrInfo **procinfo, Oid *collation, MemoryContext updateCtx)
{
char *base = NULL;
int idx = -1;
HnswNeighborArray *neighbors;
MemoryContext oldCtx = MemoryContextSwitchTo(updateCtx);
/*
* Get latest neighbors since they may have changed. Do not lock yet since
* selecting neighbors can take time. Could use optimistic locking to
* retry if another update occurs before getting exclusive lock.
*/
neighbors = HnswLoadNeighbors(element, index, m, lm, lc);
/*
* Could improve performance for vacuuming by checking neighbors against
* list of elements being deleted to find index. It's important to exclude
* already deleted elements for this since they can be replaced at any
* time.
*/
if (neighbors->length < lm)
idx = -2;
else
{
Datum q = HnswGetValue(base, element);
IndexTuple qtup = HnswPtrAccess(base, element->itup);;
LoadElementsForInsert(neighbors, q, qtup, &idx, index, procinfo, collation);
if (idx == -1)
HnswUpdateConnection(base, neighbors, newElement, distance, lm, &idx, index, procinfo, collation);
}
MemoryContextSwitchTo(oldCtx);
MemoryContextReset(updateCtx);
return idx;
}
/* /*
* Check if connection already exists * Check if connection already exists
*/ */
@@ -456,94 +336,14 @@ ConnectionExists(HnswElement e, HnswNeighborTuple ntup, int startIdx, int lm)
return false; return false;
} }
/*
* Update neighbor
*/
static void
UpdateNeighborOnDisk(HnswElement element, HnswElement newElement, int idx, int m, int lm, int lc, Relation index, bool checkExisting, bool building)
{
Buffer buf;
Page page;
GenericXLogState *state;
HnswNeighborTuple ntup;
int startIdx;
OffsetNumber offno = element->neighborOffno;
/* Register page */
buf = ReadBuffer(index, element->neighborPage);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
if (building)
{
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Get tuple */
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Calculate index for update */
startIdx = (element->level - lc) * m;
/* Check for existing connection */
if (checkExisting && ConnectionExists(newElement, ntup, startIdx, lm))
idx = -1;
else if (idx == -2)
{
/* Find free offset if still exists */
/* TODO Retry updating connections if not */
for (int j = 0; j < lm; j++)
{
if (!ItemPointerIsValid(&ntup->indextids[startIdx + j]))
{
idx = startIdx + j;
break;
}
}
}
else
idx += startIdx;
/* Make robust to issues */
if (idx >= 0 && idx < ntup->count)
{
ItemPointer indextid = &ntup->indextids[idx];
/* Update neighbor on the buffer */
ItemPointerSet(indextid, newElement->blkno, newElement->offno);
/* Commit */
if (building)
MarkBufferDirty(buf);
else
GenericXLogFinish(state);
}
else if (!building)
GenericXLogAbort(state);
UnlockReleaseBuffer(buf);
}
/* /*
* Update neighbors * Update neighbors
*/ */
void void
HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement e, int m, bool checkExisting, bool building) HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building)
{ {
char *base = NULL; char *base = NULL;
/* Use separate memory context to improve performance for larger vectors */
MemoryContext updateCtx = GenerationContextCreate(CurrentMemoryContext,
"Hnsw insert update context",
#if PG_VERSION_NUM >= 150000
128 * 1024, 128 * 1024,
#endif
128 * 1024);
for (int lc = e->level; lc >= 0; lc--) for (int lc = e->level; lc >= 0; lc--)
{ {
int lm = HnswGetLayerM(m, lc); int lm = HnswGetLayerM(m, lc);
@@ -552,20 +352,92 @@ HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, H
for (int i = 0; i < neighbors->length; i++) for (int i = 0; i < neighbors->length; i++)
{ {
HnswCandidate *hc = &neighbors->items[i]; HnswCandidate *hc = &neighbors->items[i];
Buffer buf;
Page page;
GenericXLogState *state;
HnswNeighborTuple ntup;
int idx = -1;
int startIdx;
HnswElement neighborElement = HnswPtrAccess(base, hc->element); HnswElement neighborElement = HnswPtrAccess(base, hc->element);
int idx; OffsetNumber offno = neighborElement->neighborOffno;
idx = GetUpdateIndex(neighborElement, e, hc->distance, m, lm, lc, index, procinfo, collation, updateCtx); /* Get latest neighbors since they may have changed */
/* Do not lock yet since selecting neighbors can take time */
HnswLoadNeighbors(neighborElement, index, m);
/*
* Could improve performance for vacuuming by checking neighbors
* against list of elements being deleted to find index. It's
* important to exclude already deleted elements for this since
* they can be replaced at any time.
*/
/* Select neighbors */
HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, procinfo, collation);
/* New element was not selected as a neighbor */ /* New element was not selected as a neighbor */
if (idx == -1) if (idx == -1)
continue; continue;
UpdateNeighborOnDisk(neighborElement, e, idx, m, lm, lc, index, checkExisting, building); /* Register page */
buf = ReadBuffer(index, neighborElement->neighborPage);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
if (building)
{
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Get tuple */
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Calculate index for update */
startIdx = (neighborElement->level - lc) * m;
/* Check for existing connection */
if (checkExisting && ConnectionExists(e, ntup, startIdx, lm))
idx = -1;
else if (idx == -2)
{
/* Find free offset if still exists */
/* TODO Retry updating connections if not */
for (int j = 0; j < lm; j++)
{
if (!ItemPointerIsValid(&ntup->indextids[startIdx + j]))
{
idx = startIdx + j;
break;
}
}
}
else
idx += startIdx;
/* Make robust to issues */
if (idx >= 0 && idx < ntup->count)
{
ItemPointer indextid = &ntup->indextids[idx];
/* Update neighbor on the buffer */
ItemPointerSet(indextid, e->blkno, e->offno);
/* Commit */
if (building)
MarkBufferDirty(buf);
else
GenericXLogFinish(state);
}
else if (!building)
GenericXLogAbort(state);
UnlockReleaseBuffer(buf);
} }
} }
MemoryContextDelete(updateCtx);
} }
/* /*
@@ -633,26 +505,16 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
char *base = NULL; char *base = NULL;
HnswNeighborArray *neighbors = HnswGetNeighbors(base, element, 0); HnswNeighborArray *neighbors = HnswGetNeighbors(base, element, 0);
Datum value = HnswGetValue(base, element); Datum value = HnswGetValue(base, element);
IndexTuple itup = HnswPtrAccess(base, element->itup);
TupleDesc tupdesc = RelationGetDescr(index);
for (int i = 0; i < neighbors->length; i++) for (int i = 0; i < neighbors->length; i++)
{ {
HnswCandidate *neighbor = &neighbors->items[i]; HnswCandidate *neighbor = &neighbors->items[i];
HnswElement neighborElement = HnswPtrAccess(base, neighbor->element); HnswElement neighborElement = HnswPtrAccess(base, neighbor->element);
Datum neighborValue = HnswGetValue(base, neighborElement);
if (HnswUseIndexTuple(index)) /* Exit early since ordered by distance */
{ if (!datumIsEqual(value, neighborValue, false, -1))
/* Exit early since ordered by distance */ return false;
if (!HnswIndexTupleIsEqual(itup, HnswPtrAccess(base, neighborElement->itup), tupdesc))
return false;
}
else
{
/* Exit early since ordered by distance */
if (!datumIsEqual(value, HnswGetValue(base, neighborElement), false, -1))
return false;
}
if (AddDuplicateOnDisk(index, element, neighborElement, building)) if (AddDuplicateOnDisk(index, element, neighborElement, building))
return true; return true;
@@ -665,7 +527,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
* Update graph on disk * Update graph on disk
*/ */
static void static void
UpdateGraphOnDisk(Relation index, FmgrInfo **procinfo, Oid *collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building) UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
{ {
BlockNumber newInsertPage = InvalidBlockNumber; BlockNumber newInsertPage = InvalidBlockNumber;
@@ -698,13 +560,11 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
HnswElement element; HnswElement element;
int m; int m;
int efConstruction = HnswGetEfConstruction(index); int efConstruction = HnswGetEfConstruction(index);
FmgrInfo *procinfo[2]; FmgrInfo *procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
Oid *collation; Oid collation = index->rd_indcollation[0];
LOCKMODE lockmode = ShareLock; LOCKMODE lockmode = ShareLock;
char *base = NULL; char *base = NULL;
HnswInitProcinfo(procinfo, &collation, index);
/* /*
* Get a shared lock. This allows vacuum to ensure no in-flight inserts * Get a shared lock. This allows vacuum to ensure no in-flight inserts
* before repairing graph. Use a page lock so it does not interfere with * before repairing graph. Use a page lock so it does not interfere with
@@ -717,23 +577,7 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
/* Create an element */ /* Create an element */
element = HnswInitElement(base, heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL); element = HnswInitElement(base, heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
if (HnswUseIndexTuple(index)) HnswPtrStore(base, element->value, DatumGetPointer(value));
{
/* TODO no toast */
TupleDesc tupdesc = RelationGetDescr(index);
IndexTuple itup;
bool unused;
/* TODO fix */
values[0] = value;
itup = index_form_tuple(tupdesc, values, isnull);
HnswPtrStore(base, element->itup, itup);
HnswPtrStore(base, element->value, DatumGetPointer(index_getattr(itup, 1, tupdesc, &unused)));
}
else
HnswPtrStore(base, element->value, DatumGetPointer(value));
/* Prevent concurrent inserts when likely updating entry point */ /* Prevent concurrent inserts when likely updating entry point */
if (entryPoint == NULL || element->level > entryPoint->level) if (entryPoint == NULL || element->level > entryPoint->level)
@@ -750,7 +594,7 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
} }
/* Find neighbors for element */ /* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false, false); HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false);
/* Update graph on disk */ /* Update graph on disk */
UpdateGraphOnDisk(index, procinfo, collation, element, m, efConstruction, entryPoint, building); UpdateGraphOnDisk(index, procinfo, collation, element, m, efConstruction, entryPoint, building);
@@ -770,7 +614,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
Datum value; Datum value;
const HnswTypeInfo *typeInfo = HnswGetTypeInfo(index); const HnswTypeInfo *typeInfo = HnswGetTypeInfo(index);
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
Oid *collation = index->rd_indcollation; Oid collation = index->rd_indcollation[0];
/* Detoast once for all calls */ /* Detoast once for all calls */
value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
@@ -783,10 +627,10 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC); normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
if (normprocinfo != NULL) if (normprocinfo != NULL)
{ {
if (!HnswCheckNorm(normprocinfo, collation[0], value)) if (!HnswCheckNorm(normprocinfo, collation, value))
return; return;
value = HnswNormValue(typeInfo, collation[0], value); value = HnswNormValue(typeInfo, collation, value);
} }
HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false); HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false);

View File

@@ -15,15 +15,13 @@ GetScanItems(IndexScanDesc scan, Datum q)
{ {
HnswScanOpaque so = (HnswScanOpaque) scan->opaque; HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
Relation index = scan->indexRelation; Relation index = scan->indexRelation;
FmgrInfo **procinfo = so->procinfo; FmgrInfo *procinfo = so->procinfo;
Oid *collation = so->collation; Oid collation = so->collation;
List *ep; List *ep;
List *w; List *w;
int m; int m;
HnswElement entryPoint; HnswElement entryPoint;
char *base = NULL; char *base = NULL;
bool inMemory = false;
ScanKeyData *keyData = scan->keyData;
/* Get m and entry point */ /* Get m and entry point */
HnswGetMetaPageInfo(index, &m, &entryPoint); HnswGetMetaPageInfo(index, &m, &entryPoint);
@@ -31,15 +29,15 @@ GetScanItems(IndexScanDesc scan, Datum q)
if (entryPoint == NULL) if (entryPoint == NULL)
return NIL; return NIL;
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, NULL, keyData, index, procinfo, collation, false, inMemory)); ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
for (int lc = entryPoint->level; lc >= 1; lc--) for (int lc = entryPoint->level; lc >= 1; lc--)
{ {
w = HnswSearchLayer(base, q, NULL, keyData, ep, 1, lc, index, procinfo, collation, m, false, NULL, inMemory); w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
ep = w; ep = w;
} }
return HnswSearchLayer(base, q, NULL, keyData, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, inMemory); return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
} }
/* /*
@@ -63,7 +61,7 @@ GetScanValue(IndexScanDesc scan)
/* Normalize if needed */ /* Normalize if needed */
if (so->normprocinfo != NULL) if (so->normprocinfo != NULL)
value = HnswNormValue(so->typeInfo, so->collation[0], value); value = HnswNormValue(so->typeInfo, so->collation, value);
} }
return value; return value;
@@ -88,8 +86,9 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
ALLOCSET_DEFAULT_SIZES); ALLOCSET_DEFAULT_SIZES);
/* Set support functions */ /* Set support functions */
HnswInitProcinfo(so->procinfo, &so->collation, index); so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC); so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
so->collation = index->rd_indcollation[0];
scan->opaque = so; scan->opaque = so;
@@ -161,20 +160,20 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
so->first = false; so->first = false;
#if defined(HNSW_MEMORY) #if defined(HNSW_MEMORY) && PG_VERSION_NUM >= 130000
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(so->tmpCtx, false) / 1024); elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(so->tmpCtx, false) / (1024 * 1024));
#endif #endif
} }
while (list_length(so->w) > 0) while (list_length(so->w) > 0)
{ {
char *base = NULL; char *base = NULL;
HnswSearchCandidate *hc = llast(so->w); HnswCandidate *hc = llast(so->w);
HnswElement element = HnswPtrAccess(base, hc->element); HnswElement element = HnswPtrAccess(base, hc->element);
ItemPointer heaptid; ItemPointer heaptid;
/* Move to next element if no valid heap TIDs */ /* Move to next element if no valid heap TIDs */
if (!hc->matches || element->heaptidsLength == 0) if (element->heaptidsLength == 0)
{ {
so->w = list_delete_last(so->w); so->w = list_delete_last(so->w);
continue; continue;

File diff suppressed because it is too large Load Diff

View File

@@ -189,8 +189,8 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
GenericXLogState *state; GenericXLogState *state;
int m = vacuumstate->m; int m = vacuumstate->m;
int efConstruction = vacuumstate->efConstruction; int efConstruction = vacuumstate->efConstruction;
FmgrInfo **procinfo = vacuumstate->procinfo; FmgrInfo *procinfo = vacuumstate->procinfo;
Oid *collation = vacuumstate->collation; Oid collation = vacuumstate->collation;
BufferAccessStrategy bas = vacuumstate->bas; BufferAccessStrategy bas = vacuumstate->bas;
HnswNeighborTuple ntup = vacuumstate->ntup; HnswNeighborTuple ntup = vacuumstate->ntup;
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m); Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
@@ -205,7 +205,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
element->heaptidsLength = 0; element->heaptidsLength = 0;
/* Find neighbors for element, skipping itself */ /* Find neighbors for element, skipping itself */
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true, false); HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
/* Zero memory for each element */ /* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE); MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
@@ -256,7 +256,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
LockPage(index, HNSW_UPDATE_LOCK, ShareLock); LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Load element */ /* Load element */
HnswLoadElement(highestPoint, NULL, NULL, NULL, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL); HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
/* Repair if needed */ /* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint)) if (NeedsUpdated(vacuumstate, highestPoint))
@@ -294,7 +294,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
* is outdated, this can remove connections at higher levels in * is outdated, this can remove connections at higher levels in
* the graph until they are repaired, but this should be fine. * the graph until they are repaired, but this should be fine.
*/ */
HnswLoadElement(entryPoint, NULL, NULL, NULL, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true, NULL); HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
if (NeedsUpdated(vacuumstate, entryPoint)) if (NeedsUpdated(vacuumstate, entryPoint))
{ {
@@ -370,7 +370,7 @@ RepairGraph(HnswVacuumState * vacuumstate)
/* Create an element */ /* Create an element */
element = HnswInitElementFromBlock(blkno, offno); element = HnswInitElementFromBlock(blkno, offno);
HnswLoadElementFromTuple(element, etup, false, true, index); HnswLoadElementFromTuple(element, etup, false, true);
elements = lappend(elements, element); elements = lappend(elements, element);
} }
@@ -440,7 +440,6 @@ MarkDeleted(HnswVacuumState * vacuumstate)
BlockNumber insertPage = InvalidBlockNumber; BlockNumber insertPage = InvalidBlockNumber;
Relation index = vacuumstate->index; Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas; BufferAccessStrategy bas = vacuumstate->bas;
bool useIndexTuple = HnswUseIndexTuple(index);
/* /*
* Wait for index scans to complete. Scans before this point may contain * Wait for index scans to complete. Scans before this point may contain
@@ -522,14 +521,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
/* Overwrite element */ /* Overwrite element */
etup->deleted = 1; etup->deleted = 1;
if (useIndexTuple) MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
{
IndexTuple itup = (IndexTuple) &etup->data;
MemSet(itup, 0, IndexTupleSize(itup));
}
else
MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
/* Overwrite neighbors */ /* Overwrite neighbors */
for (int i = 0; i < ntup->count; i++) for (int i = 0; i < ntup->count; i++)
@@ -581,7 +573,8 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
vacuumstate->callback_state = callback_state; vacuumstate->callback_state = callback_state;
vacuumstate->efConstruction = HnswGetEfConstruction(index); vacuumstate->efConstruction = HnswGetEfConstruction(index);
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD); vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
HnswInitProcinfo(vacuumstate->procinfo, &vacuumstate->collation, index); vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
vacuumstate->collation = index->rd_indcollation[0];
vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE); vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext, vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Hnsw vacuum temporary context", "Hnsw vacuum temporary context",

View File

@@ -26,6 +26,12 @@
#include "pgstat.h" #include "pgstat.h"
#endif #endif
#if PG_VERSION_NUM >= 130000
#define CALLBACK_ITEM_POINTER ItemPointer tid
#else
#define CALLBACK_ITEM_POINTER HeapTuple hup
#endif
#if PG_VERSION_NUM >= 140000 #if PG_VERSION_NUM >= 140000
#include "utils/backend_status.h" #include "utils/backend_status.h"
#include "utils/wait_event.h" #include "utils/wait_event.h"
@@ -90,7 +96,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
* Callback for sampling * Callback for sampling
*/ */
static void static void
SampleCallback(Relation index, ItemPointer tid, Datum *values, SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state) bool *isnull, bool tupleIsAlive, void *state)
{ {
IvfflatBuildState *buildstate = (IvfflatBuildState *) state; IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
@@ -201,12 +207,16 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
* Callback for table_index_build_scan * Callback for table_index_build_scan
*/ */
static void static void
BuildCallback(Relation index, ItemPointer tid, Datum *values, BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state) bool *isnull, bool tupleIsAlive, void *state)
{ {
IvfflatBuildState *buildstate = (IvfflatBuildState *) state; IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
MemoryContext oldCtx; MemoryContext oldCtx;
#if PG_VERSION_NUM < 130000
ItemPointer tid = &hup->t_self;
#endif
/* Skip nulls */ /* Skip nulls */
if (isnull[0]) if (isnull[0])
return; return;
@@ -325,20 +335,14 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
/* Disallow varbit since require fixed dimensions */ /* Disallow varbit since require fixed dimensions */
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID) if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
ereport(ERROR, elog(ERROR, "type not supported for ivfflat index");
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("type not supported for ivfflat index")));
/* Require column to have dimensions to be indexed */ /* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0) if (buildstate->dimensions < 0)
ereport(ERROR, elog(ERROR, "column does not have dimensions");
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("column does not have dimensions")));
if (buildstate->dimensions > buildstate->typeInfo->maxDimensions) if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
ereport(ERROR, elog(ERROR, "column cannot have more than %d dimensions for ivfflat index", buildstate->typeInfo->maxDimensions);
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("column cannot have more than %d dimensions for ivfflat index", buildstate->typeInfo->maxDimensions)));
buildstate->reltuples = 0; buildstate->reltuples = 0;
buildstate->indtuples = 0; buildstate->indtuples = 0;
@@ -351,9 +355,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
/* Require more than one dimension for spherical k-means */ /* Require more than one dimension for spherical k-means */
if (buildstate->kmeansnormprocinfo != NULL && buildstate->dimensions == 1) if (buildstate->kmeansnormprocinfo != NULL && buildstate->dimensions == 1)
ereport(ERROR, elog(ERROR, "dimensions must be greater than one for this opclass");
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions must be greater than one for this opclass")));
/* Create tuple description for sorting */ /* Create tuple description for sorting */
buildstate->tupdesc = CreateTemplateTupleDesc(3); buildstate->tupdesc = CreateTemplateTupleDesc(3);
@@ -560,20 +562,6 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
} }
#endif #endif
/*
* Initialize build sort state
*/
static Tuplesortstate *
InitBuildSortState(TupleDesc tupdesc, int memory, SortCoordinate coordinate)
{
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, memory, coordinate, false);
}
/* /*
* Within leader, wait for end of heap scan * Within leader, wait for end of heap scan
*/ */
@@ -621,6 +609,12 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
double reltuples; double reltuples;
IndexInfo *indexInfo; IndexInfo *indexInfo;
/* Sort options, which must match AssignTuples */
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
/* Initialize local tuplesort coordination state */ /* Initialize local tuplesort coordination state */
coordinate = palloc0(sizeof(SortCoordinateData)); coordinate = palloc0(sizeof(SortCoordinateData));
coordinate->isWorker = true; coordinate->isWorker = true;
@@ -633,7 +627,7 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
InitBuildState(&buildstate, ivfspool->heap, ivfspool->index, indexInfo); InitBuildState(&buildstate, ivfspool->heap, ivfspool->index, indexInfo);
memcpy(buildstate.centers->items, ivfcenters, buildstate.centers->itemsize * buildstate.centers->maxlen); memcpy(buildstate.centers->items, ivfcenters, buildstate.centers->itemsize * buildstate.centers->maxlen);
buildstate.centers->length = buildstate.centers->maxlen; buildstate.centers->length = buildstate.centers->maxlen;
ivfspool->sortstate = InitBuildSortState(buildstate.tupdesc, sortmem, coordinate); ivfspool->sortstate = tuplesort_begin_heap(buildstate.tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, sortmem, coordinate, false);
buildstate.sortstate = ivfspool->sortstate; buildstate.sortstate = ivfspool->sortstate;
scan = table_beginscan_parallel(ivfspool->heap, scan = table_beginscan_parallel(ivfspool->heap,
ParallelTableScanFromIvfflatShared(ivfshared)); ParallelTableScanFromIvfflatShared(ivfshared));
@@ -930,6 +924,12 @@ AssignTuples(IvfflatBuildState * buildstate)
int parallel_workers = 0; int parallel_workers = 0;
SortCoordinate coordinate = NULL; SortCoordinate coordinate = NULL;
/* Sort options, which must match IvfflatParallelScanAndSort */
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN); pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN);
/* Calculate parallel workers */ /* Calculate parallel workers */
@@ -950,7 +950,7 @@ AssignTuples(IvfflatBuildState * buildstate)
} }
/* Begin serial/leader tuplesort */ /* Begin serial/leader tuplesort */
buildstate->sortstate = InitBuildSortState(buildstate->tupdesc, maintenance_work_mem, coordinate); buildstate->sortstate = tuplesort_begin_heap(buildstate->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, maintenance_work_mem, coordinate, false);
/* Add tuples to sort */ /* Add tuples to sort */
if (buildstate->heap != NULL) if (buildstate->heap != NULL)
@@ -1006,10 +1006,6 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo); CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum); CreateEntryPages(buildstate, forkNum);
/* Write WAL for initialization fork since GenericXLog functions do not */
if (forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
FreeBuildState(buildstate); FreeBuildState(buildstate);
} }

View File

@@ -7,7 +7,6 @@
#include "commands/progress.h" #include "commands/progress.h"
#include "commands/vacuum.h" #include "commands/vacuum.h"
#include "ivfflat.h" #include "ivfflat.h"
#include "utils/float.h"
#include "utils/guc.h" #include "utils/guc.h"
#include "utils/selfuncs.h" #include "utils/selfuncs.h"
#include "utils/spccache.h" #include "utils/spccache.h"
@@ -27,7 +26,11 @@ IvfflatInit(void)
{ {
ivfflat_relopt_kind = add_reloption_kind(); ivfflat_relopt_kind = add_reloption_kind();
add_int_reloption(ivfflat_relopt_kind, "lists", "Number of inverted lists", add_int_reloption(ivfflat_relopt_kind, "lists", "Number of inverted lists",
IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, AccessExclusiveLock); IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes", DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
"Valid range is 1..lists.", &ivfflat_probes, "Valid range is 1..lists.", &ivfflat_probes,
@@ -69,16 +72,14 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
GenericCosts costs; GenericCosts costs;
int lists; int lists;
double ratio; double ratio;
double sequentialRatio = 0.5;
double startupPages;
double spc_seq_page_cost; double spc_seq_page_cost;
Relation index; Relation index;
/* Never use index without order */ /* Never use index without order */
if (path->indexorderbys == NULL) if (path->indexorderbys == NULL)
{ {
*indexStartupCost = get_float8_infinity(); *indexStartupCost = DBL_MAX;
*indexTotalCost = get_float8_infinity(); *indexTotalCost = DBL_MAX;
*indexSelectivity = 0; *indexSelectivity = 0;
*indexCorrelation = 0; *indexCorrelation = 0;
*indexPages = 0; *indexPages = 0;
@@ -87,8 +88,6 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
MemSet(&costs, 0, sizeof(costs)); MemSet(&costs, 0, sizeof(costs));
genericcostestimate(root, path, loop_count, &costs);
index = index_open(path->indexinfo->indexoid, NoLock); index = index_open(path->indexinfo->indexoid, NoLock);
IvfflatGetMetaPageInfo(index, &lists, NULL); IvfflatGetMetaPageInfo(index, &lists, NULL);
index_close(index, NoLock); index_close(index, NoLock);
@@ -98,26 +97,41 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
if (ratio > 1.0) if (ratio > 1.0)
ratio = 1.0; ratio = 1.0;
/*
* This gives us the subset of tuples to visit. This value is passed into
* the generic cost estimator to determine the number of pages to visit
* during the index scan.
*/
costs.numIndexTuples = path->indexinfo->tuples * ratio;
genericcostestimate(root, path, loop_count, &costs);
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost); get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
/* Change some page cost from random to sequential */
costs.indexTotalCost -= sequentialRatio * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Startup cost is cost before returning the first row */
costs.indexStartupCost = costs.indexTotalCost * ratio;
/* Adjust cost if needed since TOAST not included in seq scan cost */ /* Adjust cost if needed since TOAST not included in seq scan cost */
startupPages = costs.numIndexPages * ratio; if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
{ {
/* Change rest of page cost from random to sequential */ /* Change all page cost from random to sequential */
costs.indexStartupCost -= (1 - sequentialRatio) * startupPages * (costs.spc_random_page_cost - spc_seq_page_cost); costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Remove cost of extra pages */ /* Remove cost of extra pages */
costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost; costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
}
else
{
/* Change some page cost from random to sequential */
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
} }
*indexStartupCost = costs.indexStartupCost; /*
* If the list selectivity is lower than what is returned from the generic
* cost estimator, use that.
*/
if (ratio < costs.indexSelectivity)
costs.indexSelectivity = ratio;
/* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost;
*indexTotalCost = costs.indexTotalCost; *indexTotalCost = costs.indexTotalCost;
*indexSelectivity = costs.indexSelectivity; *indexSelectivity = costs.indexSelectivity;
*indexCorrelation = costs.indexCorrelation; *indexCorrelation = costs.indexCorrelation;
@@ -134,10 +148,23 @@ ivfflatoptions(Datum reloptions, bool validate)
{"lists", RELOPT_TYPE_INT, offsetof(IvfflatOptions, lists)}, {"lists", RELOPT_TYPE_INT, offsetof(IvfflatOptions, lists)},
}; };
#if PG_VERSION_NUM >= 130000
return (bytea *) build_reloptions(reloptions, validate, return (bytea *) build_reloptions(reloptions, validate,
ivfflat_relopt_kind, ivfflat_relopt_kind,
sizeof(IvfflatOptions), sizeof(IvfflatOptions),
tab, lengthof(tab)); tab, lengthof(tab));
#else
relopt_value *options;
int numoptions;
IvfflatOptions *rdopts;
options = parseRelOptions(reloptions, validate, ivfflat_relopt_kind, &numoptions);
rdopts = allocateReloptStruct(sizeof(IvfflatOptions), options, numoptions);
fillRelOptions((void *) rdopts, sizeof(IvfflatOptions), options, numoptions,
validate, tab, lengthof(tab));
return (bytea *) rdopts;
#endif
} }
/* /*
@@ -154,7 +181,7 @@ ivfflatvalidate(Oid opclassoid)
* *
* See https://www.postgresql.org/docs/current/index-api.html * See https://www.postgresql.org/docs/current/index-api.html
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler); PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
Datum Datum
ivfflathandler(PG_FUNCTION_ARGS) ivfflathandler(PG_FUNCTION_ARGS)
{ {
@@ -162,7 +189,9 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amstrategies = 0; amroutine->amstrategies = 0;
amroutine->amsupport = 5; amroutine->amsupport = 5;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0; amroutine->amoptsprocnum = 0;
#endif
amroutine->amcanorder = false; amroutine->amcanorder = false;
amroutine->amcanorderbyop = true; amroutine->amcanorderbyop = true;
amroutine->amcanbackward = false; /* can change direction mid-scan */ amroutine->amcanbackward = false; /* can change direction mid-scan */
@@ -175,24 +204,17 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amclusterable = false; amroutine->amclusterable = false;
amroutine->ampredlocks = false; amroutine->ampredlocks = false;
amroutine->amcanparallel = false; amroutine->amcanparallel = false;
#if PG_VERSION_NUM >= 170000
amroutine->amcanbuildparallel = true;
#endif
amroutine->amcaninclude = false; amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */ amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
#if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false;
#endif
amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL; amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
#endif
amroutine->amkeytype = InvalidOid; amroutine->amkeytype = InvalidOid;
/* Interface functions */ /* Interface functions */
amroutine->ambuild = ivfflatbuild; amroutine->ambuild = ivfflatbuild;
amroutine->ambuildempty = ivfflatbuildempty; amroutine->ambuildempty = ivfflatbuildempty;
amroutine->aminsert = ivfflatinsert; amroutine->aminsert = ivfflatinsert;
#if PG_VERSION_NUM >= 170000
amroutine->aminsertcleanup = NULL;
#endif
amroutine->ambulkdelete = ivfflatbulkdelete; amroutine->ambulkdelete = ivfflatbulkdelete;
amroutine->amvacuumcleanup = ivfflatvacuumcleanup; amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */ amroutine->amcanreturn = NULL; /* tuple not included in heapsort */

View File

@@ -253,9 +253,8 @@ typedef struct IvfflatScanOpaqueData
/* Sorting */ /* Sorting */
Tuplesortstate *sortstate; Tuplesortstate *sortstate;
TupleDesc tupdesc; TupleDesc tupdesc;
TupleTableSlot *vslot; TupleTableSlot *slot;
TupleTableSlot *mslot; bool isnull;
BufferAccessStrategy bas;
/* Support functions */ /* Support functions */
FmgrInfo *procinfo; FmgrInfo *procinfo;

View File

@@ -94,9 +94,6 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
value = IvfflatNormValue(typeInfo, collation, value); value = IvfflatNormValue(typeInfo, collation, value);
} }
/* Ensure index is valid */
IvfflatGetMetaPageInfo(index, NULL, NULL);
/* Find the insert page - sets the page and list info */ /* Find the insert page - sets the page and list info */
FindInsertPage(index, values, &insertPage, &listInfo); FindInsertPage(index, values, &insertPage, &listInfo);
Assert(BlockNumberIsValid(insertPage)); Assert(BlockNumberIsValid(insertPage));

View File

@@ -151,8 +151,12 @@ RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeI
static void static void
ShowMemoryUsage(MemoryContext context, Size estimatedSize) ShowMemoryUsage(MemoryContext context, Size estimatedSize)
{ {
#if PG_VERSION_NUM >= 130000
elog(INFO, "total memory: %zu MB", elog(INFO, "total memory: %zu MB",
MemoryContextMemAllocated(context, true) / (1024 * 1024)); MemoryContextMemAllocated(context, true) / (1024 * 1024));
#else
MemoryContextStats(context);
#endif
elog(INFO, "estimated memory: %zu MB", estimatedSize / (1024 * 1024)); elog(INFO, "estimated memory: %zu MB", estimatedSize / (1024 * 1024));
} }
#endif #endif
@@ -323,7 +327,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
newCenters->length = numCenters; newCenters->length = numCenters;
#ifdef IVFFLAT_MEMORY #ifdef IVFFLAT_MEMORY
ShowMemoryUsage(MemoryContextGetParent(CurrentMemoryContext), totalSize); ShowMemoryUsage(MemoryContextGetParent(CurrentMemoryContext));
#endif #endif
/* Pick initial centers */ /* Pick initial centers */

View File

@@ -11,23 +11,16 @@
#include "pgstat.h" #include "pgstat.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#ifdef IVFFLAT_MEMORY
#include "utils/memutils.h"
#endif
#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
/* /*
* Compare list distances * Compare list distances
*/ */
static int static int
CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg) CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{ {
if (GetScanListConst(a)->distance > GetScanListConst(b)->distance) if (((const IvfflatScanList *) a)->distance > ((const IvfflatScanList *) b)->distance)
return 1; return 1;
if (GetScanListConst(a)->distance < GetScanListConst(b)->distance) if (((const IvfflatScanList *) a)->distance < ((const IvfflatScanList *) b)->distance)
return -1; return -1;
return 0; return 0;
@@ -79,14 +72,14 @@ GetScanLists(IndexScanDesc scan, Datum value)
/* Calculate max distance */ /* Calculate max distance */
if (listCount == so->probes) if (listCount == so->probes)
maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance; maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
} }
else if (distance < maxDistance) else if (distance < maxDistance)
{ {
IvfflatScanList *scanlist; IvfflatScanList *scanlist;
/* Remove */ /* Remove */
scanlist = GetScanList(pairingheap_remove_first(so->listQueue)); scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
/* Reuse */ /* Reuse */
scanlist->startPage = list->startPage; scanlist->startPage = list->startPage;
@@ -94,7 +87,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
pairingheap_add(so->listQueue, &scanlist->ph_node); pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Update max distance */ /* Update max distance */
maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance; maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
} }
} }
@@ -113,12 +106,19 @@ GetScanItems(IndexScanDesc scan, Datum value)
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque; IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation); TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
double tuples = 0; double tuples = 0;
TupleTableSlot *slot = so->vslot; TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
/*
* Reuse same set of shared buffers for scan
*
* See postgres/src/backend/storage/buffer/README for description
*/
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
/* Search closest probes lists */ /* Search closest probes lists */
while (!pairingheap_is_empty(so->listQueue)) while (!pairingheap_is_empty(so->listQueue))
{ {
BlockNumber searchPage = GetScanList(pairingheap_remove_first(so->listQueue))->startPage; BlockNumber searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
/* Search all entry pages for list */ /* Search all entry pages for list */
while (BlockNumberIsValid(searchPage)) while (BlockNumberIsValid(searchPage))
@@ -127,7 +127,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
Page page; Page page;
OffsetNumber maxoffno; OffsetNumber maxoffno;
buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, so->bas); buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE); LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf); page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page); maxoffno = PageGetMaxOffsetNumber(page);
@@ -166,6 +166,8 @@ GetScanItems(IndexScanDesc scan, Datum value)
} }
} }
FreeAccessStrategy(bas);
if (tuples < 100) if (tuples < 100)
ereport(DEBUG1, ereport(DEBUG1,
(errmsg("index scan found few tuples"), (errmsg("index scan found few tuples"),
@@ -215,20 +217,6 @@ GetScanValue(IndexScanDesc scan)
return value; return value;
} }
/*
* Initialize scan sort state
*/
static Tuplesortstate *
InitScanSortState(TupleDesc tupdesc)
{
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
}
/* /*
* Prepare for an index scan * Prepare for an index scan
*/ */
@@ -239,6 +227,10 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
IvfflatScanOpaque so; IvfflatScanOpaque so;
int lists; int lists;
int dimensions; int dimensions;
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
int probes = ivfflat_probes; int probes = ivfflat_probes;
scan = RelationGetIndexScan(index, nkeys, norderbys); scan = RelationGetIndexScan(index, nkeys, norderbys);
@@ -266,18 +258,9 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
/* Prep sort */ /* Prep sort */
so->sortstate = InitScanSortState(so->tupdesc); so->sortstate = tuplesort_begin_heap(so->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
/* Need separate slots for puttuple and gettuple */ so->slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
so->vslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
so->mslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
/*
* Reuse same set of shared buffers for scan
*
* See postgres/src/backend/storage/buffer/README for description
*/
so->bas = GetAccessStrategy(BAS_BULKREAD);
so->listQueue = pairingheap_allocate(CompareLists, scan); so->listQueue = pairingheap_allocate(CompareLists, scan);
@@ -294,8 +277,10 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
{ {
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque; IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
#if PG_VERSION_NUM >= 130000
if (!so->first) if (!so->first)
tuplesort_reset(so->sortstate); tuplesort_reset(so->sortstate);
#endif
so->first = true; so->first = true;
pairingheap_reset(so->listQueue); pairingheap_reset(so->listQueue);
@@ -342,19 +327,14 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
IvfflatBench("GetScanItems", GetScanItems(scan, value)); IvfflatBench("GetScanItems", GetScanItems(scan, value));
so->first = false; so->first = false;
#if defined(IVFFLAT_MEMORY)
elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
#endif
/* Clean up if we allocated a new value */ /* Clean up if we allocated a new value */
if (value != scan->orderByData->sk_argument) if (value != scan->orderByData->sk_argument)
pfree(DatumGetPointer(value)); pfree(DatumGetPointer(value));
} }
if (tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL)) if (tuplesort_gettupleslot(so->sortstate, true, false, so->slot, NULL))
{ {
bool isnull; ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
scan->xs_heaptid = *heaptid; scan->xs_heaptid = *heaptid;
scan->xs_recheck = false; scan->xs_recheck = false;
@@ -375,10 +355,6 @@ ivfflatendscan(IndexScanDesc scan)
pairingheap_free(so->listQueue); pairingheap_free(so->listQueue);
tuplesort_end(so->sortstate); tuplesort_end(so->sortstate);
FreeAccessStrategy(so->bas);
FreeTupleDesc(so->tupdesc);
/* TODO Free vslot and mslot without freeing TupleDesc */
pfree(so); pfree(so);
scan->opaque = NULL; scan->opaque = NULL;

View File

@@ -170,11 +170,7 @@ IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions)
page = BufferGetPage(buf); page = BufferGetPage(buf);
metap = IvfflatPageGetMeta(page); metap = IvfflatPageGetMeta(page);
if (unlikely(metap->magicNumber != IVFFLAT_MAGIC_NUMBER)) *lists = metap->lists;
elog(ERROR, "ivfflat index is not valid");
if (lists != NULL)
*lists = metap->lists;
if (dimensions != NULL) if (dimensions != NULL)
*dimensions = metap->dimensions; *dimensions = metap->dimensions;
@@ -342,7 +338,7 @@ IvfflatGetTypeInfo(Relation index)
return (const IvfflatTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid)); return (const IvfflatTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid));
} }
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_halfvec_support); PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflat_halfvec_support);
Datum Datum
ivfflat_halfvec_support(PG_FUNCTION_ARGS) ivfflat_halfvec_support(PG_FUNCTION_ARGS)
{ {
@@ -357,7 +353,7 @@ ivfflat_halfvec_support(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(&typeInfo); PG_RETURN_POINTER(&typeInfo);
}; };
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support); PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflat_bit_support);
Datum Datum
ivfflat_bit_support(PG_FUNCTION_ARGS) ivfflat_bit_support(PG_FUNCTION_ARGS)
{ {

View File

@@ -3,7 +3,6 @@
#include <limits.h> #include <limits.h>
#include <math.h> #include <math.h>
#include "catalog/pg_type.h"
#include "common/string.h" #include "common/string.h"
#include "fmgr.h" #include "fmgr.h"
#include "halfutils.h" #include "halfutils.h"
@@ -12,7 +11,6 @@
#include "sparsevec.h" #include "sparsevec.h"
#include "utils/array.h" #include "utils/array.h"
#include "utils/builtins.h" #include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "vector.h" #include "vector.h"
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
@@ -190,7 +188,7 @@ CompareIndices(const void *a, const void *b)
/* /*
* Convert textual representation to internal representation * Convert textual representation to internal representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_in);
Datum Datum
sparsevec_in(PG_FUNCTION_ARGS) sparsevec_in(PG_FUNCTION_ARGS)
{ {
@@ -411,7 +409,7 @@ sparsevec_in(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to textual representation * Convert internal representation to textual representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_out); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_out);
Datum Datum
sparsevec_out(PG_FUNCTION_ARGS) sparsevec_out(PG_FUNCTION_ARGS)
{ {
@@ -464,7 +462,7 @@ sparsevec_out(PG_FUNCTION_ARGS)
/* /*
* Convert type modifier * Convert type modifier
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_typmod_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_typmod_in);
Datum Datum
sparsevec_typmod_in(PG_FUNCTION_ARGS) sparsevec_typmod_in(PG_FUNCTION_ARGS)
{ {
@@ -495,7 +493,7 @@ sparsevec_typmod_in(PG_FUNCTION_ARGS)
/* /*
* Convert external binary representation to internal representation * Convert external binary representation to internal representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_recv); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_recv);
Datum Datum
sparsevec_recv(PG_FUNCTION_ARGS) sparsevec_recv(PG_FUNCTION_ARGS)
{ {
@@ -547,7 +545,7 @@ sparsevec_recv(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to the external binary representation * Convert internal representation to the external binary representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_send); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_send);
Datum Datum
sparsevec_send(PG_FUNCTION_ARGS) sparsevec_send(PG_FUNCTION_ARGS)
{ {
@@ -574,7 +572,7 @@ sparsevec_send(PG_FUNCTION_ARGS)
* Convert sparse vector to sparse vector * Convert sparse vector to sparse vector
* This is needed to check the type modifier * This is needed to check the type modifier
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec);
Datum Datum
sparsevec(PG_FUNCTION_ARGS) sparsevec(PG_FUNCTION_ARGS)
{ {
@@ -589,7 +587,7 @@ sparsevec(PG_FUNCTION_ARGS)
/* /*
* Convert dense vector to sparse vector * Convert dense vector to sparse vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_sparsevec); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_sparsevec);
Datum Datum
vector_to_sparsevec(PG_FUNCTION_ARGS) vector_to_sparsevec(PG_FUNCTION_ARGS)
{ {
@@ -632,7 +630,7 @@ vector_to_sparsevec(PG_FUNCTION_ARGS)
/* /*
* Convert half vector to sparse vector * Convert half vector to sparse vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_sparsevec); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_sparsevec);
Datum Datum
halfvec_to_sparsevec(PG_FUNCTION_ARGS) halfvec_to_sparsevec(PG_FUNCTION_ARGS)
{ {
@@ -672,137 +670,6 @@ halfvec_to_sparsevec(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
/*
* Convert array to sparse vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_sparsevec);
Datum
array_to_sparsevec(PG_FUNCTION_ARGS)
{
ArrayType *array = PG_GETARG_ARRAYTYPE_P(0);
int32 typmod = PG_GETARG_INT32(1);
SparseVector *result;
int16 typlen;
bool typbyval;
char typalign;
Datum *elemsp;
int nelemsp;
int nnz = 0;
float *values;
int j = 0;
if (ARR_NDIM(array) > 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("array must be 1-D")));
if (ARR_HASNULL(array) && array_contains_nulls(array))
ereport(ERROR,
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
errmsg("array must not contain nulls")));
get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, NULL, &nelemsp);
CheckDim(nelemsp);
CheckExpectedDim(typmod, nelemsp);
#ifdef _MSC_VER
/* /fp:fast may not propagate +/-Infinity or NaN */
#define IS_NOT_ZERO(v) (isnan((float) (v)) || isinf((float) (v)) || ((float) (v)) != 0)
#else
#define IS_NOT_ZERO(v) (((float) (v)) != 0)
#endif
if (ARR_ELEMTYPE(array) == INT4OID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DatumGetInt32(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT8OID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DatumGetFloat8(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT4OID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DatumGetFloat4(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == NUMERICOID)
{
for (int i = 0; i < nelemsp; i++)
nnz += IS_NOT_ZERO(DirectFunctionCall1(numeric_float4, elemsp[i]));
}
else
{
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("unsupported array type")));
}
result = InitSparseVector(nelemsp, nnz);
values = SPARSEVEC_VALUES(result);
#define PROCESS_ARRAY_ELEM(elem) \
do { \
float v = (float) (elem); \
if (IS_NOT_ZERO(v)) { \
/* Safety check */ \
if (j >= result->nnz) \
elog(ERROR, "safety check failed"); \
result->indices[j] = i; \
values[j] = v; \
j++; \
} \
} while (0)
if (ARR_ELEMTYPE(array) == INT4OID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetInt32(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT8OID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetFloat8(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == FLOAT4OID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetFloat4(elemsp[i]));
}
else if (ARR_ELEMTYPE(array) == NUMERICOID)
{
for (int i = 0; i < nelemsp; i++)
PROCESS_ARRAY_ELEM(DatumGetFloat4(DirectFunctionCall1(numeric_float4, elemsp[i])));
}
else
{
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("unsupported array type")));
}
#undef PROCESS_ARRAY_ELEM
#undef IS_NOT_ZERO
/*
* Free allocation from deconstruct_array. Do not free individual elements
* when pass-by-reference since they point to original array.
*/
pfree(elemsp);
if (j != result->nnz)
elog(ERROR, "correctness check failed");
/* Check elements */
for (int i = 0; i < result->nnz; i++)
CheckElement(values[i]);
PG_RETURN_POINTER(result);
}
/* /*
* Get the L2 squared distance between sparse vectors * Get the L2 squared distance between sparse vectors
*/ */
@@ -854,7 +721,7 @@ SparsevecL2SquaredDistance(SparseVector * a, SparseVector * b)
/* /*
* Get the L2 distance between sparse vectors * Get the L2 distance between sparse vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_distance);
Datum Datum
sparsevec_l2_distance(PG_FUNCTION_ARGS) sparsevec_l2_distance(PG_FUNCTION_ARGS)
{ {
@@ -870,7 +737,7 @@ sparsevec_l2_distance(PG_FUNCTION_ARGS)
* Get the L2 squared distance between sparse vectors * Get the L2 squared distance between sparse vectors
* This saves a sqrt calculation * This saves a sqrt calculation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_squared_distance);
Datum Datum
sparsevec_l2_squared_distance(PG_FUNCTION_ARGS) sparsevec_l2_squared_distance(PG_FUNCTION_ARGS)
{ {
@@ -921,7 +788,7 @@ SparsevecInnerProduct(SparseVector * a, SparseVector * b)
/* /*
* Get the inner product of two sparse vectors * Get the inner product of two sparse vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_inner_product);
Datum Datum
sparsevec_inner_product(PG_FUNCTION_ARGS) sparsevec_inner_product(PG_FUNCTION_ARGS)
{ {
@@ -936,7 +803,7 @@ sparsevec_inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the negative inner product of two sparse vectors * Get the negative inner product of two sparse vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_negative_inner_product);
Datum Datum
sparsevec_negative_inner_product(PG_FUNCTION_ARGS) sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
{ {
@@ -951,7 +818,7 @@ sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the cosine distance between two sparse vectors * Get the cosine distance between two sparse vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_cosine_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cosine_distance);
Datum Datum
sparsevec_cosine_distance(PG_FUNCTION_ARGS) sparsevec_cosine_distance(PG_FUNCTION_ARGS)
{ {
@@ -996,7 +863,7 @@ sparsevec_cosine_distance(PG_FUNCTION_ARGS)
/* /*
* Get the L1 distance between two sparse vectors * Get the L1 distance between two sparse vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l1_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l1_distance);
Datum Datum
sparsevec_l1_distance(PG_FUNCTION_ARGS) sparsevec_l1_distance(PG_FUNCTION_ARGS)
{ {
@@ -1045,7 +912,7 @@ sparsevec_l1_distance(PG_FUNCTION_ARGS)
/* /*
* Get the L2 norm of a sparse vector * Get the L2 norm of a sparse vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_norm); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_norm);
Datum Datum
sparsevec_l2_norm(PG_FUNCTION_ARGS) sparsevec_l2_norm(PG_FUNCTION_ARGS)
{ {
@@ -1063,7 +930,7 @@ sparsevec_l2_norm(PG_FUNCTION_ARGS)
/* /*
* Normalize a sparse vector with the L2 norm * Normalize a sparse vector with the L2 norm
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_l2_normalize); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_l2_normalize);
Datum Datum
sparsevec_l2_normalize(PG_FUNCTION_ARGS) sparsevec_l2_normalize(PG_FUNCTION_ARGS)
{ {
@@ -1173,7 +1040,7 @@ sparsevec_cmp_internal(SparseVector * a, SparseVector * b)
/* /*
* Less than * Less than
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_lt); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_lt);
Datum Datum
sparsevec_lt(PG_FUNCTION_ARGS) sparsevec_lt(PG_FUNCTION_ARGS)
{ {
@@ -1186,7 +1053,7 @@ sparsevec_lt(PG_FUNCTION_ARGS)
/* /*
* Less than or equal * Less than or equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_le); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_le);
Datum Datum
sparsevec_le(PG_FUNCTION_ARGS) sparsevec_le(PG_FUNCTION_ARGS)
{ {
@@ -1199,7 +1066,7 @@ sparsevec_le(PG_FUNCTION_ARGS)
/* /*
* Equal * Equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_eq); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_eq);
Datum Datum
sparsevec_eq(PG_FUNCTION_ARGS) sparsevec_eq(PG_FUNCTION_ARGS)
{ {
@@ -1212,7 +1079,7 @@ sparsevec_eq(PG_FUNCTION_ARGS)
/* /*
* Not equal * Not equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_ne); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_ne);
Datum Datum
sparsevec_ne(PG_FUNCTION_ARGS) sparsevec_ne(PG_FUNCTION_ARGS)
{ {
@@ -1225,7 +1092,7 @@ sparsevec_ne(PG_FUNCTION_ARGS)
/* /*
* Greater than or equal * Greater than or equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_ge); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_ge);
Datum Datum
sparsevec_ge(PG_FUNCTION_ARGS) sparsevec_ge(PG_FUNCTION_ARGS)
{ {
@@ -1238,7 +1105,7 @@ sparsevec_ge(PG_FUNCTION_ARGS)
/* /*
* Greater than * Greater than
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_gt); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_gt);
Datum Datum
sparsevec_gt(PG_FUNCTION_ARGS) sparsevec_gt(PG_FUNCTION_ARGS)
{ {
@@ -1251,7 +1118,7 @@ sparsevec_gt(PG_FUNCTION_ARGS)
/* /*
* Compare sparse vectors * Compare sparse vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_cmp); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_cmp);
Datum Datum
sparsevec_cmp(PG_FUNCTION_ARGS) sparsevec_cmp(PG_FUNCTION_ARGS)
{ {

View File

@@ -26,6 +26,11 @@
#include "varatt.h" #include "varatt.h"
#endif #endif
#if PG_VERSION_NUM < 130000
#define TYPALIGN_DOUBLE 'd'
#define TYPALIGN_INT 'i'
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1) #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1)) #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -155,10 +160,28 @@ CheckStateArray(ArrayType *statearray, const char *caller)
return (float8 *) ARR_DATA_PTR(statearray); return (float8 *) ARR_DATA_PTR(statearray);
} }
#if PG_VERSION_NUM < 120003
static pg_noinline void
float_overflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: overflow")));
}
static pg_noinline void
float_underflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: underflow")));
}
#endif
/* /*
* Convert textual representation to internal representation * Convert textual representation to internal representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
Datum Datum
vector_in(PG_FUNCTION_ARGS) vector_in(PG_FUNCTION_ARGS)
{ {
@@ -271,7 +294,7 @@ vector_in(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to textual representation * Convert internal representation to textual representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_out); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
Datum Datum
vector_out(PG_FUNCTION_ARGS) vector_out(PG_FUNCTION_ARGS)
{ {
@@ -325,7 +348,7 @@ PrintVector(char *msg, Vector * vector)
/* /*
* Convert type modifier * Convert type modifier
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_typmod_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
Datum Datum
vector_typmod_in(PG_FUNCTION_ARGS) vector_typmod_in(PG_FUNCTION_ARGS)
{ {
@@ -356,7 +379,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
/* /*
* Convert external binary representation to internal representation * Convert external binary representation to internal representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_recv); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
Datum Datum
vector_recv(PG_FUNCTION_ARGS) vector_recv(PG_FUNCTION_ARGS)
{ {
@@ -390,7 +413,7 @@ vector_recv(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to the external binary representation * Convert internal representation to the external binary representation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_send); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
Datum Datum
vector_send(PG_FUNCTION_ARGS) vector_send(PG_FUNCTION_ARGS)
{ {
@@ -410,7 +433,7 @@ vector_send(PG_FUNCTION_ARGS)
* Convert vector to vector * Convert vector to vector
* This is needed to check the type modifier * This is needed to check the type modifier
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
Datum Datum
vector(PG_FUNCTION_ARGS) vector(PG_FUNCTION_ARGS)
{ {
@@ -425,7 +448,7 @@ vector(PG_FUNCTION_ARGS)
/* /*
* Convert array to vector * Convert array to vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_vector); PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
Datum Datum
array_to_vector(PG_FUNCTION_ARGS) array_to_vector(PG_FUNCTION_ARGS)
{ {
@@ -499,7 +522,7 @@ array_to_vector(PG_FUNCTION_ARGS)
/* /*
* Convert vector to float4[] * Convert vector to float4[]
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_float4); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
Datum Datum
vector_to_float4(PG_FUNCTION_ARGS) vector_to_float4(PG_FUNCTION_ARGS)
{ {
@@ -523,7 +546,7 @@ vector_to_float4(PG_FUNCTION_ARGS)
/* /*
* Convert half vector to vector * Convert half vector to vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_vector); PGDLLEXPORT PG_FUNCTION_INFO_V1(halfvec_to_vector);
Datum Datum
halfvec_to_vector(PG_FUNCTION_ARGS) halfvec_to_vector(PG_FUNCTION_ARGS)
{ {
@@ -561,7 +584,7 @@ VectorL2SquaredDistance(int dim, float *ax, float *bx)
/* /*
* Get the L2 distance between vectors * Get the L2 distance between vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
Datum Datum
l2_distance(PG_FUNCTION_ARGS) l2_distance(PG_FUNCTION_ARGS)
{ {
@@ -577,7 +600,7 @@ l2_distance(PG_FUNCTION_ARGS)
* Get the L2 squared distance between vectors * Get the L2 squared distance between vectors
* This saves a sqrt calculation * This saves a sqrt calculation
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_l2_squared_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
Datum Datum
vector_l2_squared_distance(PG_FUNCTION_ARGS) vector_l2_squared_distance(PG_FUNCTION_ARGS)
{ {
@@ -604,7 +627,7 @@ VectorInnerProduct(int dim, float *ax, float *bx)
/* /*
* Get the inner product of two vectors * Get the inner product of two vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
Datum Datum
inner_product(PG_FUNCTION_ARGS) inner_product(PG_FUNCTION_ARGS)
{ {
@@ -619,7 +642,7 @@ inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the negative inner product of two vectors * Get the negative inner product of two vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_negative_inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
Datum Datum
vector_negative_inner_product(PG_FUNCTION_ARGS) vector_negative_inner_product(PG_FUNCTION_ARGS)
{ {
@@ -653,7 +676,7 @@ VectorCosineSimilarity(int dim, float *ax, float *bx)
/* /*
* Get the cosine distance between two vectors * Get the cosine distance between two vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(cosine_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
Datum Datum
cosine_distance(PG_FUNCTION_ARGS) cosine_distance(PG_FUNCTION_ARGS)
{ {
@@ -685,7 +708,7 @@ cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality * Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm) * Assumes inputs are unit vectors (skips norm)
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_spherical_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
Datum Datum
vector_spherical_distance(PG_FUNCTION_ARGS) vector_spherical_distance(PG_FUNCTION_ARGS)
{ {
@@ -722,7 +745,7 @@ VectorL1Distance(int dim, float *ax, float *bx)
/* /*
* Get the L1 distance between two vectors * Get the L1 distance between two vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l1_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(l1_distance);
Datum Datum
l1_distance(PG_FUNCTION_ARGS) l1_distance(PG_FUNCTION_ARGS)
{ {
@@ -737,7 +760,7 @@ l1_distance(PG_FUNCTION_ARGS)
/* /*
* Get the dimensions of a vector * Get the dimensions of a vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_dims); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
Datum Datum
vector_dims(PG_FUNCTION_ARGS) vector_dims(PG_FUNCTION_ARGS)
{ {
@@ -749,7 +772,7 @@ vector_dims(PG_FUNCTION_ARGS)
/* /*
* Get the L2 norm of a vector * Get the L2 norm of a vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_norm); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
Datum Datum
vector_norm(PG_FUNCTION_ARGS) vector_norm(PG_FUNCTION_ARGS)
{ {
@@ -767,7 +790,7 @@ vector_norm(PG_FUNCTION_ARGS)
/* /*
* Normalize a vector with the L2 norm * Normalize a vector with the L2 norm
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_normalize); PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_normalize);
Datum Datum
l2_normalize(PG_FUNCTION_ARGS) l2_normalize(PG_FUNCTION_ARGS)
{ {
@@ -806,7 +829,7 @@ l2_normalize(PG_FUNCTION_ARGS)
/* /*
* Add vectors * Add vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_add); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
Datum Datum
vector_add(PG_FUNCTION_ARGS) vector_add(PG_FUNCTION_ARGS)
{ {
@@ -839,7 +862,7 @@ vector_add(PG_FUNCTION_ARGS)
/* /*
* Subtract vectors * Subtract vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_sub); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
Datum Datum
vector_sub(PG_FUNCTION_ARGS) vector_sub(PG_FUNCTION_ARGS)
{ {
@@ -872,7 +895,7 @@ vector_sub(PG_FUNCTION_ARGS)
/* /*
* Multiply vectors * Multiply vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_mul); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_mul);
Datum Datum
vector_mul(PG_FUNCTION_ARGS) vector_mul(PG_FUNCTION_ARGS)
{ {
@@ -908,7 +931,7 @@ vector_mul(PG_FUNCTION_ARGS)
/* /*
* Concatenate vectors * Concatenate vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_concat); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_concat);
Datum Datum
vector_concat(PG_FUNCTION_ARGS) vector_concat(PG_FUNCTION_ARGS)
{ {
@@ -932,7 +955,7 @@ vector_concat(PG_FUNCTION_ARGS)
/* /*
* Quantize a vector * Quantize a vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(binary_quantize); PGDLLEXPORT PG_FUNCTION_INFO_V1(binary_quantize);
Datum Datum
binary_quantize(PG_FUNCTION_ARGS) binary_quantize(PG_FUNCTION_ARGS)
{ {
@@ -950,7 +973,7 @@ binary_quantize(PG_FUNCTION_ARGS)
/* /*
* Get a subvector * Get a subvector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(subvector); PGDLLEXPORT PG_FUNCTION_INFO_V1(subvector);
Datum Datum
subvector(PG_FUNCTION_ARGS) subvector(PG_FUNCTION_ARGS)
{ {
@@ -1024,7 +1047,7 @@ vector_cmp_internal(Vector * a, Vector * b)
/* /*
* Less than * Less than
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_lt); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
Datum Datum
vector_lt(PG_FUNCTION_ARGS) vector_lt(PG_FUNCTION_ARGS)
{ {
@@ -1037,7 +1060,7 @@ vector_lt(PG_FUNCTION_ARGS)
/* /*
* Less than or equal * Less than or equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_le); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
Datum Datum
vector_le(PG_FUNCTION_ARGS) vector_le(PG_FUNCTION_ARGS)
{ {
@@ -1050,7 +1073,7 @@ vector_le(PG_FUNCTION_ARGS)
/* /*
* Equal * Equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_eq); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
Datum Datum
vector_eq(PG_FUNCTION_ARGS) vector_eq(PG_FUNCTION_ARGS)
{ {
@@ -1063,7 +1086,7 @@ vector_eq(PG_FUNCTION_ARGS)
/* /*
* Not equal * Not equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ne); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
Datum Datum
vector_ne(PG_FUNCTION_ARGS) vector_ne(PG_FUNCTION_ARGS)
{ {
@@ -1076,7 +1099,7 @@ vector_ne(PG_FUNCTION_ARGS)
/* /*
* Greater than or equal * Greater than or equal
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ge); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
Datum Datum
vector_ge(PG_FUNCTION_ARGS) vector_ge(PG_FUNCTION_ARGS)
{ {
@@ -1089,7 +1112,7 @@ vector_ge(PG_FUNCTION_ARGS)
/* /*
* Greater than * Greater than
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_gt); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
Datum Datum
vector_gt(PG_FUNCTION_ARGS) vector_gt(PG_FUNCTION_ARGS)
{ {
@@ -1102,7 +1125,7 @@ vector_gt(PG_FUNCTION_ARGS)
/* /*
* Compare vectors * Compare vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_cmp); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
Datum Datum
vector_cmp(PG_FUNCTION_ARGS) vector_cmp(PG_FUNCTION_ARGS)
{ {
@@ -1115,7 +1138,7 @@ vector_cmp(PG_FUNCTION_ARGS)
/* /*
* Accumulate vectors * Accumulate vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_accum); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_accum);
Datum Datum
vector_accum(PG_FUNCTION_ARGS) vector_accum(PG_FUNCTION_ARGS)
{ {
@@ -1176,7 +1199,7 @@ vector_accum(PG_FUNCTION_ARGS)
/* /*
* Combine vectors or half vectors (also used for halfvec_combine) * Combine vectors or half vectors (also used for halfvec_combine)
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_combine); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_combine);
Datum Datum
vector_combine(PG_FUNCTION_ARGS) vector_combine(PG_FUNCTION_ARGS)
{ {
@@ -1247,7 +1270,7 @@ vector_combine(PG_FUNCTION_ARGS)
/* /*
* Average vectors * Average vectors
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_avg); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_avg);
Datum Datum
vector_avg(PG_FUNCTION_ARGS) vector_avg(PG_FUNCTION_ARGS)
{ {
@@ -1281,7 +1304,7 @@ vector_avg(PG_FUNCTION_ARGS)
/* /*
* Convert sparse vector to dense vector * Convert sparse vector to dense vector
*/ */
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_vector); PGDLLEXPORT PG_FUNCTION_INFO_V1(sparsevec_to_vector);
Datum Datum
sparsevec_to_vector(PG_FUNCTION_ARGS) sparsevec_to_vector(PG_FUNCTION_ARGS)
{ {

View File

@@ -20,11 +20,4 @@ Vector *InitVector(int dim);
void PrintVector(char *msg, Vector * vector); void PrintVector(char *msg, Vector * vector);
int vector_cmp_internal(Vector * a, Vector * b); int vector_cmp_internal(Vector * a, Vector * b);
/* TODO Move to better place */
#if PG_VERSION_NUM >= 160000
#define FUNCTION_PREFIX
#else
#define FUNCTION_PREFIX PGDLLEXPORT
#endif
#endif #endif

View File

@@ -208,62 +208,6 @@ SELECT '{1:1e-8}/1'::sparsevec::halfvec;
[0] [0]
(1 row) (1 row)
SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
sparsevec
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
sparsevec
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
ERROR: expected 5 dimensions, not 6
SELECT '{NULL}'::real[]::sparsevec;
ERROR: array must not contain nulls
SELECT '{NaN}'::real[]::sparsevec;
ERROR: NaN not allowed in sparsevec
SELECT '{Infinity}'::real[]::sparsevec;
ERROR: infinite value not allowed in sparsevec
SELECT '{-Infinity}'::real[]::sparsevec;
ERROR: infinite value not allowed in sparsevec
SELECT '{}'::real[]::sparsevec;
ERROR: sparsevec must have at least 1 dimension
SELECT '{{1}}'::real[]::sparsevec;
ERROR: array must be 1-D
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n; SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
ERROR: vector cannot have more than 16000 dimensions ERROR: vector cannot have more than 16000 dimensions
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n; SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

3
test/expected/vector.out Normal file
View File

@@ -0,0 +1,3 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION vector" to load this file. \quit
Use "CREATE EXTENSION vector" to load this file.

View File

@@ -1,672 +0,0 @@
SELECT '[1,2,3]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::vector;
vector
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "[hello,1]"
LINE 1: SELECT '[hello,1]'::vector;
^
SELECT '[NaN,1]'::vector;
ERROR: NaN not allowed in vector
LINE 1: SELECT '[NaN,1]'::vector;
^
SELECT '[Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[Infinity,1]'::vector;
^
SELECT '[-Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[-Infinity,1]'::vector;
^
SELECT '[1.5e38,-1.5e38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e+38,-1.5e+38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e-38,-1.5e-38]'::vector;
vector
--------------------
[1.5e-38,-1.5e-38]
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: "4e38" is out of range for type vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[-4e38,1]'::vector;
ERROR: "-4e38" is out of range for type vector
LINE 1: SELECT '[-4e38,1]'::vector;
^
SELECT '[1e-46,1]'::vector;
vector
--------
[0,1]
(1 row)
SELECT '[-1e-46,1]'::vector;
vector
--------
[-0,1]
(1 row)
SELECT '[1,2,3'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector;
^
SELECT '[1,2,3]9'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::vector;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::vector;
ERROR: invalid input syntax for type vector: "1,2,3"
LINE 1: SELECT '1,2,3'::vector;
^
DETAIL: Vector contents must start with "[".
SELECT ''::vector;
ERROR: invalid input syntax for type vector: ""
LINE 1: SELECT ''::vector;
^
DETAIL: Vector contents must start with "[".
SELECT '['::vector;
ERROR: invalid input syntax for type vector: "["
LINE 1: SELECT '['::vector;
^
SELECT '[ '::vector;
ERROR: invalid input syntax for type vector: "[ "
LINE 1: SELECT '[ '::vector;
^
SELECT '[,'::vector;
ERROR: invalid input syntax for type vector: "[,"
LINE 1: SELECT '[,'::vector;
^
SELECT '[]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[]'::vector;
^
SELECT '[ ]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[ ]'::vector;
^
SELECT '[,]'::vector;
ERROR: invalid input syntax for type vector: "[,]"
LINE 1: SELECT '[,]'::vector;
^
SELECT '[1,]'::vector;
ERROR: invalid input syntax for type vector: "[1,]"
LINE 1: SELECT '[1,]'::vector;
^
SELECT '[1a]'::vector;
ERROR: invalid input syntax for type vector: "[1a]"
LINE 1: SELECT '[1a]'::vector;
^
SELECT '[1,,3]'::vector;
ERROR: invalid input syntax for type vector: "[1,,3]"
LINE 1: SELECT '[1,,3]'::vector;
^
SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
^
SELECT '[1,2,3]'::vector('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::vector('a');
^
SELECT '[1,2,3]'::vector(0);
ERROR: dimensions for type vector must be at least 1
LINE 1: SELECT '[1,2,3]'::vector(0);
^
SELECT '[1,2,3]'::vector(16001);
ERROR: dimensions for type vector cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::vector(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::vector(2)[];
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector + '[4,5,6]';
?column?
----------
[5,7,9]
(1 row)
SELECT '[3e38]'::vector + '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2]'::vector + '[3]';
ERROR: different vector dimensions 2 and 1
SELECT '[1,2,3]'::vector - '[4,5,6]';
?column?
------------
[-3,-3,-3]
(1 row)
SELECT '[-3e38]'::vector - '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2]'::vector - '[3]';
ERROR: different vector dimensions 2 and 1
SELECT '[1,2,3]'::vector * '[4,5,6]';
?column?
-----------
[4,10,18]
(1 row)
SELECT '[1e37]'::vector * '[1e37]';
ERROR: value out of range: overflow
SELECT '[1e-37]'::vector * '[1e-37]';
ERROR: value out of range: underflow
SELECT '[1,2]'::vector * '[3]';
ERROR: different vector dimensions 2 and 1
SELECT '[1,2,3]'::vector || '[4,5]';
?column?
-------------
[1,2,3,4,5]
(1 row)
SELECT array_fill(0, ARRAY[16000])::vector || '[1]';
ERROR: vector cannot have more than 16000 dimensions
SELECT '[1,2,3]'::vector < '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector < '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector = '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector = '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector > '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector > '[1,2]';
?column?
----------
t
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
vector_cmp
------------
0
(1 row)
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[1,2]', '[2,3,4]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[2,3]', '[1,2,3]');
vector_cmp
------------
1
(1 row)
SELECT vector_dims('[1,2,3]'::vector);
vector_dims
-------------
3
(1 row)
SELECT round(vector_norm('[1,1]')::numeric, 5);
round
---------
1.41421
(1 row)
SELECT vector_norm('[3,4]');
vector_norm
-------------
5
(1 row)
SELECT vector_norm('[0,1]');
vector_norm
-------------
1
(1 row)
SELECT vector_norm('[3e37,4e37]')::real;
vector_norm
-------------
5e+37
(1 row)
SELECT vector_norm('[0,0]');
vector_norm
-------------
0
(1 row)
SELECT vector_norm('[2]');
vector_norm
-------------
2
(1 row)
SELECT l2_distance('[0,0]'::vector, '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::vector, '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
l2_distance
-------------
Infinity
(1 row)
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::vector, '[1,1,1,1,1,1,1,4,5]');
l2_distance
-------------
5
(1 row)
SELECT '[0,0]'::vector <-> '[3,4]';
?column?
----------
5
(1 row)
SELECT inner_product('[1,2]'::vector, '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT inner_product('[3e38]'::vector, '[3e38]');
inner_product
---------------
Infinity
(1 row)
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::vector, '[1,2,3,4,5,6,7,8,9]');
inner_product
---------------
45
(1 row)
SELECT '[1,2]'::vector <#> '[3,4]';
?column?
----------
-11
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
cosine_distance
-----------------
2
(1 row)
SELECT '[1,2]'::vector <=> '[2,4]';
?column?
----------
0
(1 row)
SELECT l1_distance('[0,0]'::vector, '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::vector, '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
l1_distance
-------------
Infinity
(1 row)
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
l1_distance
-------------
0
(1 row)
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[0,3,2,5,4,7,6,9,8]');
l1_distance
-------------
9
(1 row)
SELECT '[0,0]'::vector <+> '[3,4]';
?column?
----------
7
(1 row)
SELECT l2_normalize('[3,4]'::vector);
l2_normalize
--------------
[0.6,0.8]
(1 row)
SELECT l2_normalize('[3,0]'::vector);
l2_normalize
--------------
[1,0]
(1 row)
SELECT l2_normalize('[0,0.1]'::vector);
l2_normalize
--------------
[0,1]
(1 row)
SELECT l2_normalize('[0,0]'::vector);
l2_normalize
--------------
[0,0]
(1 row)
SELECT l2_normalize('[3e38]'::vector);
l2_normalize
--------------
[1]
(1 row)
SELECT binary_quantize('[1,0,-1]'::vector);
binary_quantize
-----------------
100
(1 row)
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
binary_quantize
-----------------
01001110101
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
subvector
-----------
[1,2,3]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
subvector
-----------
[3,4]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, -1, 3);
subvector
-----------
[1]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 3, 9);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 0);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 2147483647, 10);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2147483647);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, -2147483644, 2147483647);
subvector
-----------
[1,2]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
avg
-----
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: expected 2 dimensions, not 1
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
avg
---------
[3e+38]
(1 row)
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
ERROR: vector cannot have more than 16000 dimensions
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
sum
-----
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: different vector dimensions 2 and 1
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
ERROR: value out of range: overflow

View File

@@ -1,11 +0,0 @@
package PostgreSQL::Test::Cluster;
use PostgresNode;
sub new
{
my ($class, $name) = @_;
return get_new_node($name);
}
1;

View File

@@ -1,5 +0,0 @@
package PostgreSQL::Test::Utils;
use TestLib;
1;

View File

@@ -0,0 +1,8 @@
use PostgreSQL::Test::Cluster;
sub get_new_node
{
return PostgreSQL::Test::Cluster->new(@_);
}
1;

3
test/perl/TestLib.pm Normal file
View File

@@ -0,0 +1,3 @@
use PostgreSQL::Test::Utils;
1;

View File

@@ -58,22 +58,6 @@ SELECT '{}/16001'::sparsevec::halfvec;
SELECT '{1:65520}/1'::sparsevec::halfvec; SELECT '{1:65520}/1'::sparsevec::halfvec;
SELECT '{1:1e-8}/1'::sparsevec::halfvec; SELECT '{1:1e-8}/1'::sparsevec::halfvec;
SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
SELECT '{NULL}'::real[]::sparsevec;
SELECT '{NaN}'::real[]::sparsevec;
SELECT '{Infinity}'::real[]::sparsevec;
SELECT '{-Infinity}'::real[]::sparsevec;
SELECT '{}'::real[]::sparsevec;
SELECT '{{1}}'::real[]::sparsevec;
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n; SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n; SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -2,9 +2,9 @@
# Test generic xlog record work for ivfflat index replication. # Test generic xlog record work for ivfflat index replication.
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 32; my $dim = 32;
@@ -49,7 +49,7 @@ sub test_index_replay
my $array_sql = join(",", ('random()') x $dim); my $array_sql = join(",", ('random()') x $dim);
# Initialize primary node # Initialize primary node
$node_primary = PostgreSQL::Test::Cluster->new('primary'); $node_primary = get_new_node('primary');
$node_primary->init(allows_streaming => 1); $node_primary->init(allows_streaming => 1);
if ($dim > 32) if ($dim > 32)
{ {
@@ -67,7 +67,7 @@ my $backup_name = 'my_backup';
$node_primary->backup($backup_name); $node_primary->backup($backup_name);
# Create streaming replica linking to primary # Create streaming replica linking to primary
$node_replica = PostgreSQL::Test::Cluster->new('replica'); $node_replica = get_new_node('replica');
$node_replica->init_from_backup($node_primary, $backup_name, has_streaming => 1); $node_replica->init_from_backup($node_primary, $backup_name, has_streaming => 1);
$node_replica->start; $node_replica->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 3; my $dim = 3;
@@ -15,7 +15,7 @@ for (1 .. $dim)
my $array_sql = join(", ", @r); my $array_sql = join(", ", @r);
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -49,7 +49,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -48,7 +48,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 768; my $dim = 768;
@@ -9,7 +9,7 @@ my $dim = 768;
my $array_sql = join(",", ('random()') x $dim); my $array_sql = join(",", ('random()') x $dim);
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 3; my $dim = 3;
@@ -11,7 +11,7 @@ my $limit = 20;
my $array_sql = join(",", ('random()') x $dim); my $array_sql = join(",", ('random()') x $dim);
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
@@ -94,7 +94,8 @@ like($explain, qr/Seq Scan/);
$explain = $node->safe_psql("postgres", qq( $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query'; EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query';
)); ));
like($explain, qr/Seq Scan/); # TODO Do not use index
like($explain, qr/Index Scan using idx/);
# Test attribute index # Test attribute index
$node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);"); $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
@@ -109,6 +110,7 @@ $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING ivfflat (v v
$explain = $node->safe_psql("postgres", qq( $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit; EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
)); ));
like($explain, qr/Index Scan using partial_idx/); # TODO Use partial index
like($explain, qr/Index Scan using idx/);
done_testing(); done_testing();

View File

@@ -2,9 +2,9 @@
# Test generic xlog record work for hnsw index replication. # Test generic xlog record work for hnsw index replication.
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 32; my $dim = 32;
@@ -49,7 +49,7 @@ sub test_index_replay
my $array_sql = join(",", ('random()') x $dim); my $array_sql = join(",", ('random()') x $dim);
# Initialize primary node # Initialize primary node
$node_primary = PostgreSQL::Test::Cluster->new('primary'); $node_primary = get_new_node('primary');
$node_primary->init(allows_streaming => 1); $node_primary->init(allows_streaming => 1);
if ($dim > 32) if ($dim > 32)
{ {
@@ -67,7 +67,7 @@ my $backup_name = 'my_backup';
$node_primary->backup($backup_name); $node_primary->backup($backup_name);
# Create streaming replica linking to primary # Create streaming replica linking to primary
$node_replica = PostgreSQL::Test::Cluster->new('replica'); $node_replica = get_new_node('replica');
$node_replica->init_from_backup($node_primary, $backup_name, has_streaming => 1); $node_replica->init_from_backup($node_primary, $backup_name, has_streaming => 1);
$node_replica->start; $node_replica->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 3; my $dim = 3;
@@ -15,7 +15,7 @@ for (1 .. $dim)
my $array_sql = join(", ", @r); my $array_sql = join(", ", @r);
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -47,7 +47,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
@@ -91,7 +91,7 @@ for my $i (0 .. $#operators)
)); ));
# Test approximate results # Test approximate results
my $min = $operator eq "<#>" ? 0.97 : 0.99; my $min = $operator eq "<#>" ? 0.98 : 0.99;
test_recall($min, $operator); test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;"); $node->safe_psql("postgres", "DROP INDEX idx;");

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -47,7 +47,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
@@ -100,7 +100,7 @@ for my $i (0 .. $#operators)
} }
# Test approximate results # Test approximate results
my $min = $operator eq "<#>" ? 0.97 : 0.99; my $min = $operator eq "<#>" ? 0.98 : 0.99;
test_recall($min, $operator); test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;"); $node->safe_psql("postgres", "DROP INDEX idx;");

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -48,7 +48,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Ensures elements and neighbors on both same and different pages # Ensures elements and neighbors on both same and different pages
@@ -10,7 +10,7 @@ my $dim = 1900;
my $array_sql = join(",", ('random()') x $dim); my $array_sql = join(",", ('random()') x $dim);
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 3; my $dim = 3;
@@ -11,20 +11,16 @@ my $limit = 20;
my $array_sql = join(",", ('random()') x $dim); my $array_sql = join(",", ('random()') x $dim);
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
# Create table and index # Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;"); $node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);"); $node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4, t text);");
$node->safe_psql("postgres", "CREATE TABLE cat (i int4 PRIMARY KEY, t text, b boolean);");
$node->safe_psql("postgres", $node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;" "INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc, 'test ' || i FROM generate_series(1, 10000) i;"
); );
$node->safe_psql("postgres",
"INSERT INTO cat SELECT i, 'cat ' || i, i % 5 = 0 FROM generate_series(1, $nc) i;"
);
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);"); $node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "ANALYZE tst;"); $node->safe_psql("postgres", "ANALYZE tst;");
@@ -41,7 +37,8 @@ my $c = int(rand() * $nc);
my $explain = $node->safe_psql("postgres", qq( my $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit; EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
)); ));
like($explain, qr/Seq Scan/); # TODO Do not use index
like($explain, qr/Index Scan using idx/);
# Test attribute filtering with few rows removed # Test attribute filtering with few rows removed
$explain = $node->safe_psql("postgres", qq( $explain = $node->safe_psql("postgres", qq(
@@ -59,7 +56,8 @@ like($explain, qr/Index Scan using idx/);
$explain = $node->safe_psql("postgres", qq( $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE c < 1 ORDER BY v <-> '$query' LIMIT $limit; EXPLAIN ANALYZE SELECT i FROM tst WHERE c < 1 ORDER BY v <-> '$query' LIMIT $limit;
)); ));
like($explain, qr/Seq Scan/); # TODO Do not use index
like($explain, qr/Index Scan using idx/);
# Test attribute filtering with few rows removed like # Test attribute filtering with few rows removed like
$explain = $node->safe_psql("postgres", qq( $explain = $node->safe_psql("postgres", qq(
@@ -98,25 +96,13 @@ $explain = $node->safe_psql("postgres", qq(
)); ));
like($explain, qr/Seq Scan/); like($explain, qr/Seq Scan/);
# Test join
$explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c ORDER BY v <-> '$query' LIMIT $limit;
));
like($explain, qr/Index Scan using idx/);
# Test join with attribute filtering
$explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT cat.t FROM cat INNER JOIN tst ON cat.i = tst.c WHERE cat.b = 't' ORDER BY v <-> '$query' LIMIT $limit;
));
like($explain, qr/Index Scan using idx/);
# Test attribute index # Test attribute index
$node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);"); $node->safe_psql("postgres", "CREATE INDEX attribute_idx ON tst (c);");
$explain = $node->safe_psql("postgres", qq( $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit; EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $c ORDER BY v <-> '$query' LIMIT $limit;
)); ));
# Use attribute index # TODO Use attribute index
like($explain, qr/Bitmap Index Scan on attribute_idx/); like($explain, qr/Index Scan using idx/);
# Test partial index # Test partial index
$node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);"); $node->safe_psql("postgres", "CREATE INDEX partial_idx ON tst USING hnsw (v vector_l2_ops) WHERE (c = $c);");

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
@@ -53,7 +53,7 @@ sub test_aggregate
else else
{ {
# Does not raise overflow error in this instance due to loss of precision # Does not raise overflow error in this instance due to loss of precision
is($res, "[24576,24576,49152]"); is($res, "[24576,24576,49152]")
} }
} }

View File

@@ -1,13 +1,13 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $dim = 1024; my $dim = 1024;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -51,7 +51,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -51,7 +51,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -51,7 +51,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -48,7 +48,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -48,7 +48,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -48,7 +48,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -47,7 +47,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
@@ -91,7 +91,7 @@ for my $i (0 .. $#operators)
)); ));
# Test approximate results # Test approximate results
my $min = $operator eq "<#>" ? 0.97 : 0.99; my $min = $operator eq "<#>" ? 0.98 : 0.99;
test_recall($min, $operator); test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;"); $node->safe_psql("postgres", "DROP INDEX idx;");

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -47,7 +47,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;
@@ -100,7 +100,7 @@ for my $i (0 .. $#operators)
} }
# Test approximate results # Test approximate results
my $min = $operator eq "<#>" ? 0.97 : 0.99; my $min = $operator eq "<#>" ? 0.98 : 0.99;
test_recall($min, $operator); test_recall($min, $operator);
$node->safe_psql("postgres", "DROP INDEX idx;"); $node->safe_psql("postgres", "DROP INDEX idx;");

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -48,7 +48,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -51,7 +51,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,14 +1,14 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
my $array_sql = join(",", ('floor(random() * 2)::int - 1') x 3); my $array_sql = join(",", ('floor(random() * 2)::int - 1') x 3);
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -10,7 +10,7 @@ my $dim = 5;
my $array_sql = join(",", ('floor(random() * 4)::int - 2') x $dim); my $array_sql = join(",", ('floor(random() * 4)::int - 2') x $dim);
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,7 +1,7 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
my $node; my $node;
@@ -51,7 +51,7 @@ sub test_recall
} }
# Initialize node # Initialize node
$node = PostgreSQL::Test::Cluster->new('node'); $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,11 +1,11 @@
use strict; use strict;
use warnings FATAL => 'all'; use warnings;
use PostgreSQL::Test::Cluster; use PostgresNode;
use PostgreSQL::Test::Utils; use TestLib;
use Test::More; use Test::More;
# Initialize node # Initialize node
my $node = PostgreSQL::Test::Cluster->new('node'); my $node = get_new_node('node');
$node->init; $node->init;
$node->start; $node->start;

View File

@@ -1,42 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v sparsevec(100000));");
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v sparsevec_l2_ops);");
for (1 .. 3)
{
for (1 .. 100)
{
my @elements;
my %indices;
for (1 .. int(rand() * 100))
{
my $index = int(rand() * (100000 - 1)) + 1;
if (!exists($indices{$index}))
{
my $value = rand();
push(@elements, "$index:$value");
$indices{$index} = 1;
}
}
my $embedding = "{" . join(",", @elements) . "}/100000";
$node->safe_psql("postgres", "INSERT INTO tst (v) VALUES ('$embedding');");
}
$node->safe_psql("postgres", "DELETE FROM tst WHERE i % 2 = 0;");
$node->safe_psql("postgres", "VACUUM tst;");
is(1, 1);
}
done_testing();

View File

@@ -1,60 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my @dims = (384, 1536);
my $limit = 10;
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
for my $dim (@dims)
{
my $array_sql = join(",", ('random()') x $dim);
# Create table and index
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 2000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "ANALYZE tst;");
# Generate query
my @r = ();
for (1 .. $dim)
{
push(@r, rand());
}
my $query = "[" . join(",", @r) . "]";
my $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
));
like($explain, qr/Index Scan using idx/);
# 3x the rows are needed for distance filters
# since the planner uses DEFAULT_INEQ_SEL for the selectivity (should be 1)
# Recreate index for performance
$node->safe_psql("postgres", "DROP INDEX idx;");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(2001, 6000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "ANALYZE tst;");
$explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
));
like($explain, qr/Index Scan using idx/);
$node->safe_psql("postgres", "DROP TABLE tst;");
}
done_testing();

View File

@@ -1,50 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my @dims = (384, 1536);
my $limit = 10;
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
for my $dim (@dims)
{
my $array_sql = join(",", ('random()') x $dim);
# Create table and index
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[$array_sql] FROM generate_series(1, 5000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX idx ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 5);");
$node->safe_psql("postgres", "ANALYZE tst;");
# Generate query
my @r = ();
for (1 .. $dim)
{
push(@r, rand());
}
my $query = "[" . join(",", @r) . "]";
my $explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v <-> '$query' LIMIT $limit;
));
like($explain, qr/Index Scan using idx/);
$explain = $node->safe_psql("postgres", qq(
EXPLAIN ANALYZE SELECT i FROM tst WHERE v <-> '$query' < 1 ORDER BY v <-> '$query' LIMIT $limit;
));
like($explain, qr/Index Scan using idx/);
$node->safe_psql("postgres", "DROP TABLE tst;");
}
done_testing();

View File

@@ -1,109 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $node;
my @queries = ();
my @cs = ();
my @expected;
my $limit = 20;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
my $nc = 50;
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
EXPLAIN ANALYZE SELECT i FROM tst WHERE c = $cs[0] ORDER BY v $operator '$queries[0]' LIMIT $limit;
));
like($explain, qr/Index Cond/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT i FROM tst WHERE c = $cs[$i] ORDER BY v $operator '$queries[$i]' LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids;
is(scalar(@actual_ids), $limit);
my @expected_ids = split("\n", $expected[$i]);
foreach (@expected_ids)
{
if (exists($actual_set{$_}))
{
$correct++;
}
$total++;
}
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim), c int4);");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[$array_sql], i % $nc FROM generate_series(1, 20000) i;"
);
# Generate queries
for (1 .. 20)
{
my @r = ();
for (1 .. $dim)
{
push(@r, rand());
}
push(@queries, "[" . join(",", @r) . "]");
push(@cs, int(rand() * $nc));
}
# Get exact results
@expected = ();
for my $i (0 .. $#queries)
{
my $res = $node->safe_psql("postgres", "SELECT i FROM tst WHERE c = $cs[$i] ORDER BY v <-> '$queries[$i]' LIMIT $limit;");
push(@expected, $res);
}
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops, c);");
# Test recall
test_recall(0.99, '<->');
# Test vacuum
$node->safe_psql("postgres", "DELETE FROM tst WHERE c > 5;");
$node->safe_psql("postgres", "VACUUM tst;");
# Test columns
my ($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (c);");
like($stderr, qr/first column must be a vector/);
($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (c, v vector_l2_ops);");
like($stderr, qr/first column must be a vector/);
($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops, c, c);");
like($stderr, qr/index cannot have more than two columns/);
($ret, $stdout, $stderr) = $node->psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops, v vector_l2_ops);");
like($stderr, qr/column 2 cannot be a vector/);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods' comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.7.4' default_version = '0.6.2'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true